From e6731a95747c9f900372462c65d6f57ca3c44bb8 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 12 Aug 2019 13:41:59 -0700 Subject: [PATCH 01/55] Redo test Signed-off-by: Nicolas Flacco --- .../extensions/clusters/redis/redis_cluster.cc | 5 +++-- .../extensions/clusters/redis/redis_cluster.h | 1 + .../filters/network/common/redis/BUILD | 1 + .../filters/network/common/redis/client.h | 11 ++++++++++- .../filters/network/common/redis/client_impl.cc | 17 ++++++++++------- .../filters/network/common/redis/client_impl.h | 10 +++++++--- .../network/redis_proxy/conn_pool_impl.cc | 4 ++-- .../network/redis_proxy/conn_pool_impl.h | 10 +--------- .../extensions/health_checkers/redis/redis.cc | 7 ++++--- source/extensions/health_checkers/redis/redis.h | 1 + test/extensions/clusters/redis/BUILD | 1 + .../clusters/redis/redis_cluster_test.cc | 2 +- .../filters/network/common/redis/BUILD | 1 + .../network/common/redis/client_impl_test.cc | 14 ++++++++++++-- .../network/redis_proxy/conn_pool_impl_test.cc | 2 +- .../health_checkers/redis/redis_test.cc | 2 +- 16 files changed, 57 insertions(+), 32 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index bf5160805944..aae80d48965f 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -34,7 +34,8 @@ RedisCluster::RedisCluster( : Config::Utility::translateClusterHosts(cluster.hosts())), local_info_(factory_context.localInfo()), random_(factory_context.random()), redis_discovery_session_(*this, redis_client_factory), lb_factory_(std::move(lb_factory)), - api_(api) { + api_(api), + redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(info()->statsScope()), POOL_HISTOGRAM(info()->statsScope()))} { const auto& locality_lb_endpoints = load_assignment_.endpoints(); for (const auto& locality_lb_endpoint : locality_lb_endpoints) { for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { @@ -240,7 +241,7 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = client_factory_.create(host, dispatcher_, *this); + client->client_ = client_factory_.create(host, dispatcher_, *this, parent_.redis_cluster_stats_); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index d7b776353671..55f54850fa13 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -251,6 +251,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { envoy::api::v2::core::DataSource auth_password_datasource_; Api::Api& api_; + Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats redis_cluster_stats_; }; class RedisClusterFactory : public Upstream::ConfigurableClusterFactoryBase< diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 1757c2a7b9b4..ac070361b193 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -58,6 +58,7 @@ envoy_cc_library( ":client_interface", ":codec_lib", "//include/envoy/router:router_interface", + "//include/envoy/stats:timespan", "//include/envoy/thread_local:thread_local_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/buffer:buffer_lib", diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 4e041ad33552..88b80f98669a 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -13,6 +13,15 @@ namespace Common { namespace Redis { namespace Client { +#define REDIS_CLUSTER_STATS(COUNTER, HISTOGRAM) \ + COUNTER(upstream_cx_drained) \ + COUNTER(max_upstream_unknown_connections_reached) \ + HISTOGRAM(upstream_rq_time) + +struct RedisClusterStats { + REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_HISTOGRAM_STRUCT) +}; + /** * A handle to an outbound request. */ @@ -174,7 +183,7 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) PURE; + const Config& config, RedisClusterStats& redis_cluster_stats) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 192177dca7a1..ee6cbca36b8a 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -23,10 +23,10 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config) { + const Config& config, RedisClusterStats& redis_cluster_stats) { std::unique_ptr client( - new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config)); + new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_cluster_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -36,11 +36,13 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche } ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) + EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), - flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })) { + flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), + redis_cluster_stats_(redis_cluster_stats), + time_source_(dispatcher.timeSource()) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -161,6 +163,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ASSERT(!pending_requests_.empty()); PendingRequest& request = pending_requests_.front(); const bool canceled = request.canceled_; + request.request_timer_->complete(); PoolCallbacks& callbacks = request.callbacks_; // We need to ensure the request is popped before calling the callback, since the callback might @@ -201,7 +204,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) - : parent_(parent), callbacks_(callbacks) { + : parent_(parent), callbacks_(callbacks), request_timer_(std::make_unique(parent_.redis_cluster_stats_.upstream_rq_time_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); @@ -223,9 +226,9 @@ void ClientImpl::PendingRequest::cancel() { ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, - Event::Dispatcher& dispatcher, const Config& config) { + Event::Dispatcher& dispatcher, const Config& config, RedisClusterStats& redis_cluster_stats) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config); + config, redis_cluster_stats); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index ffeb0ee458d8..add88c30b2fd 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -3,6 +3,7 @@ #include #include "envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.h" +#include "envoy/stats/timespan.h" #include "envoy/thread_local/thread_local.h" #include "envoy/upstream/cluster_manager.h" @@ -63,7 +64,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config); + const Config& config, RedisClusterStats& redis_cluster_stats); ~ClientImpl() override; @@ -101,10 +102,11 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl& parent_; PoolCallbacks& callbacks_; bool canceled_{}; + Stats::TimespanPtr request_timer_; }; ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, - DecoderFactory& decoder_factory, const Config& config); + DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); @@ -127,13 +129,15 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr connect_or_op_timer_; bool connected_{}; Event::TimerPtr flush_timer_; + RedisClusterStats redis_cluster_stats_; + Envoy::TimeSource& time_source_; }; class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) override; + const Config& config, RedisClusterStats& redis_cluster_stats) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 44f2634aa89a..551dbe030862 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -27,7 +27,7 @@ InstanceImpl::InstanceImpl( Api::Api& api, Stats::ScopePtr&& stats_scope) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(*stats_scope_))} { + POOL_COUNTER(*stats_scope_), POOL_HISTOGRAM(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -196,7 +196,7 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_); + client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, parent_.redis_cluster_stats_); client->redis_client_->addConnectionCallbacks(*client); if (!auth_password_.empty()) { // Send an AUTH command to the upstream server. diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 91ea51f3e752..07decf76535b 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -37,14 +37,6 @@ namespace ConnPool { // TODO(mattklein123): Circuit breaking // TODO(rshriram): Fault injection -#define REDIS_CLUSTER_STATS(COUNTER) \ - COUNTER(upstream_cx_drained) \ - COUNTER(max_upstream_unknown_connections_reached) - -struct RedisClusterStats { - REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT) -}; - class InstanceImpl : public Instance { public: InstanceImpl( @@ -131,7 +123,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ConfigImpl config_; Api::Api& api_; Stats::ScopePtr stats_scope_; - RedisClusterStats redis_cluster_stats_; + Common::Redis::Client::RedisClusterStats redis_cluster_stats_; }; } // namespace ConnPool diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 9f130824639c..f461e0925fe1 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -12,8 +12,9 @@ RedisHealthChecker::RedisHealthChecker( Upstream::HealthCheckEventLoggerPtr&& event_logger, Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : HealthCheckerImplBase(cluster, config, dispatcher, runtime, random, std::move(event_logger)), - client_factory_(client_factory), key_(redis_config.key()) { - if (!key_.empty()) { + client_factory_(client_factory), key_(redis_config.key()), + redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(cluster.info()->statsScope()), POOL_HISTOGRAM(cluster.info()->statsScope()))} { + if (!key_.empty()) { type_ = Type::Exists; } else { type_ = Type::Ping; @@ -51,7 +52,7 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this); + client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, parent_.redis_cluster_stats_); client_->addConnectionCallbacks(*this); } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 346b27e9b951..f3890f194255 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -113,6 +113,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; Type type_; const std::string key_; + Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats redis_cluster_stats_; }; } // namespace RedisHealthChecker diff --git a/test/extensions/clusters/redis/BUILD b/test/extensions/clusters/redis/BUILD index b990f1df93fe..a247c8fc2069 100644 --- a/test/extensions/clusters/redis/BUILD +++ b/test/extensions/clusters/redis/BUILD @@ -23,6 +23,7 @@ envoy_extension_cc_test( "//source/common/upstream:upstream_lib", "//source/extensions/clusters/redis:redis_cluster", "//source/extensions/clusters/redis:redis_cluster_lb", + "//source/extensions/filters/network/common/redis:client_interface", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 4d60412503ff..d065ba2c05a6 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -61,7 +61,7 @@ class RedisClusterTest : public testing::Test, // ClientFactory Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/BUILD b/test/extensions/filters/network/common/redis/BUILD index 8b0f46b1600f..ed87543857ac 100644 --- a/test/extensions/filters/network/common/redis/BUILD +++ b/test/extensions/filters/network/common/redis/BUILD @@ -54,6 +54,7 @@ envoy_cc_test( "//source/common/upstream:upstream_lib", "//source/extensions/filters/network/common/redis:client_lib", "//test/mocks/network:network_mocks", + "//test/mocks/stats:stats_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", ], diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 064c740dacc6..357b38ba07a8 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -43,6 +43,8 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF return Common::Redis::DecoderPtr{decoder_}; } + RedisClientImplTest() : redis_cluster_stats_(RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats_), POOL_HISTOGRAM(fake_stats_))}) {} + ~RedisClientImplTest() override { client_.reset(); @@ -77,7 +79,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, noDelay(true)); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_); + *config_, redis_cluster_stats_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -114,6 +116,8 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Network::ReadFilterSharedPtr upstream_read_filter_; std::unique_ptr config_; ClientPtr client_; + Stats::IsolatedStoreImpl fake_stats_; + RedisClusterStats redis_cluster_stats_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { @@ -278,6 +282,7 @@ TEST_F(RedisClientImplTest, Basic) { EXPECT_EQ(2UL, host_->cluster_.stats_.upstream_rq_active_.value()); EXPECT_EQ(2UL, host_->stats_.rq_total_.value()); EXPECT_EQ(2UL, host_->stats_.rq_active_.value()); + Buffer::OwnedImpl fake_data; EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { @@ -876,7 +881,12 @@ TEST(RedisClientFactoryImplTest, Basic) { EXPECT_CALL(*host, createConnection_(_, _)).WillOnce(Return(conn_info)); NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); - ClientPtr client = factory.create(host, dispatcher, config); + + // Cluster stats + Stats::IsolatedStoreImpl fake_stats; + RedisClusterStats redis_cluster_stats = RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats), POOL_HISTOGRAM(fake_stats))}; + + ClientPtr client = factory.create(host, dispatcher, config, redis_cluster_stats); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index ee16cfaa5015..27c410d9d5dd 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -159,7 +159,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Common::Redis::Client::Config&) override { + const Common::Redis::Client::Config&, Common::Redis::Client::RedisClusterStats&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index d5741a158eb4..db65aef6d77a 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -126,7 +126,7 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From 35b6128f67d8e88d4f6667af7463d08a1b250334 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 12 Aug 2019 13:45:03 -0700 Subject: [PATCH 02/55] formatting Signed-off-by: Nicolas Flacco --- .../extensions/clusters/redis/redis_cluster.cc | 7 ++++--- .../filters/network/common/redis/client_impl.cc | 17 ++++++++++------- .../filters/network/common/redis/client_impl.h | 3 ++- .../network/redis_proxy/conn_pool_impl.cc | 6 ++++-- .../extensions/health_checkers/redis/redis.cc | 11 +++++++---- .../clusters/redis/redis_cluster_test.cc | 3 ++- .../network/common/redis/client_impl_test.cc | 8 +++++--- .../network/redis_proxy/conn_pool_impl_test.cc | 3 ++- .../health_checkers/redis/redis_test.cc | 3 ++- 9 files changed, 38 insertions(+), 23 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index aae80d48965f..93665c4b2342 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -34,8 +34,8 @@ RedisCluster::RedisCluster( : Config::Utility::translateClusterHosts(cluster.hosts())), local_info_(factory_context.localInfo()), random_(factory_context.random()), redis_discovery_session_(*this, redis_client_factory), lb_factory_(std::move(lb_factory)), - api_(api), - redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(info()->statsScope()), POOL_HISTOGRAM(info()->statsScope()))} { + api_(api), redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(info()->statsScope()), + POOL_HISTOGRAM(info()->statsScope()))} { const auto& locality_lb_endpoints = load_assignment_.endpoints(); for (const auto& locality_lb_endpoint : locality_lb_endpoints) { for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { @@ -241,7 +241,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = client_factory_.create(host, dispatcher_, *this, parent_.redis_cluster_stats_); + client->client_ = + client_factory_.create(host, dispatcher_, *this, parent_.redis_cluster_stats_); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index ee6cbca36b8a..f698716141f1 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -25,8 +25,8 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats) { - std::unique_ptr client( - new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_cluster_stats)); + std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), + decoder_factory, config, redis_cluster_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -36,13 +36,13 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche } ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats) + EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, + RedisClusterStats& redis_cluster_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), - redis_cluster_stats_(redis_cluster_stats), - time_source_(dispatcher.timeSource()) { + redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -204,7 +204,9 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) - : parent_(parent), callbacks_(callbacks), request_timer_(std::make_unique(parent_.redis_cluster_stats_.upstream_rq_time_, parent_.time_source_)) { + : parent_(parent), callbacks_(callbacks), + request_timer_(std::make_unique( + parent_.redis_cluster_stats_.upstream_rq_time_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); @@ -226,7 +228,8 @@ void ClientImpl::PendingRequest::cancel() { ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, - Event::Dispatcher& dispatcher, const Config& config, RedisClusterStats& redis_cluster_stats) { + Event::Dispatcher& dispatcher, const Config& config, + RedisClusterStats& redis_cluster_stats) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, config, redis_cluster_stats); } diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index add88c30b2fd..6cb209a16244 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -106,7 +106,8 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne }; ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, - DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats); + DecoderFactory& decoder_factory, const Config& config, + RedisClusterStats& redis_cluster_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 551dbe030862..e2c14e9db803 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -27,7 +27,8 @@ InstanceImpl::InstanceImpl( Api::Api& api, Stats::ScopePtr&& stats_scope) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(*stats_scope_), POOL_HISTOGRAM(*stats_scope_))} { + POOL_COUNTER(*stats_scope_), + POOL_HISTOGRAM(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -196,7 +197,8 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, parent_.redis_cluster_stats_); + client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, + parent_.redis_cluster_stats_); client->redis_client_->addConnectionCallbacks(*client); if (!auth_password_.empty()) { // Send an AUTH command to the upstream server. diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index f461e0925fe1..55c35aa1727a 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -12,9 +12,11 @@ RedisHealthChecker::RedisHealthChecker( Upstream::HealthCheckEventLoggerPtr&& event_logger, Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : HealthCheckerImplBase(cluster, config, dispatcher, runtime, random, std::move(event_logger)), - client_factory_(client_factory), key_(redis_config.key()), - redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(cluster.info()->statsScope()), POOL_HISTOGRAM(cluster.info()->statsScope()))} { - if (!key_.empty()) { + client_factory_(client_factory), + key_(redis_config.key()), redis_cluster_stats_{REDIS_CLUSTER_STATS( + POOL_COUNTER(cluster.info()->statsScope()), + POOL_HISTOGRAM(cluster.info()->statsScope()))} { + if (!key_.empty()) { type_ = Type::Exists; } else { type_ = Type::Ping; @@ -52,7 +54,8 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, parent_.redis_cluster_stats_); + client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, + parent_.redis_cluster_stats_); client_->addConnectionCallbacks(*this); } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index d065ba2c05a6..06d6e023ba68 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -61,7 +61,8 @@ class RedisClusterTest : public testing::Test, // ClientFactory Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&, Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 357b38ba07a8..c2bbd4790388 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -43,7 +43,9 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF return Common::Redis::DecoderPtr{decoder_}; } - RedisClientImplTest() : redis_cluster_stats_(RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats_), POOL_HISTOGRAM(fake_stats_))}) {} + RedisClientImplTest() + : redis_cluster_stats_(RedisClusterStats{ + REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats_), POOL_HISTOGRAM(fake_stats_))}) {} ~RedisClientImplTest() override { client_.reset(); @@ -282,7 +284,6 @@ TEST_F(RedisClientImplTest, Basic) { EXPECT_EQ(2UL, host_->cluster_.stats_.upstream_rq_active_.value()); EXPECT_EQ(2UL, host_->stats_.rq_total_.value()); EXPECT_EQ(2UL, host_->stats_.rq_active_.value()); - Buffer::OwnedImpl fake_data; EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { @@ -884,7 +885,8 @@ TEST(RedisClientFactoryImplTest, Basic) { // Cluster stats Stats::IsolatedStoreImpl fake_stats; - RedisClusterStats redis_cluster_stats = RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats), POOL_HISTOGRAM(fake_stats))}; + RedisClusterStats redis_cluster_stats = + RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats), POOL_HISTOGRAM(fake_stats))}; ClientPtr client = factory.create(host, dispatcher, config, redis_cluster_stats); client->close(); diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 27c410d9d5dd..4d24474c1182 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -159,7 +159,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Common::Redis::Client::Config&, Common::Redis::Client::RedisClusterStats&) override { + const Common::Redis::Client::Config&, + Common::Redis::Client::RedisClusterStats&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index db65aef6d77a..d01bfd7ba66c 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -126,7 +126,8 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&, Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From bad7ea4c9b5ec01a5b60f716a9217047b01f5df0 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 12 Aug 2019 17:38:06 -0700 Subject: [PATCH 03/55] fix ws Signed-off-by: Nicolas Flacco --- .../extensions/filters/network/common/redis/client_impl_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index c2bbd4790388..c177f9f06969 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -882,8 +882,6 @@ TEST(RedisClientFactoryImplTest, Basic) { EXPECT_CALL(*host, createConnection_(_, _)).WillOnce(Return(conn_info)); NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); - - // Cluster stats Stats::IsolatedStoreImpl fake_stats; RedisClusterStats redis_cluster_stats = RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats), POOL_HISTOGRAM(fake_stats))}; From a1288f1a859718c8b8eefc35b8dade6507e068ac Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 13 Aug 2019 10:01:06 -0700 Subject: [PATCH 04/55] add docs Signed-off-by: Nicolas Flacco --- docs/root/intro/arch_overview/other_protocols/redis.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 22650a4264ad..515120bc6511 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -89,6 +89,7 @@ Every Redis cluster has its own extra statistics tree rooted at *cluster.. max_upstream_unknown_connections_reached, Counter, Total number of times that an upstream connection to an unknown host is not created after redirection having reached the connection pool's max_upstream_unknown_connections limit upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed + upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests Supported commands ------------------ From 44cc44d176770f36a36dd30232f6fdc3011e4a1c Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 14 Aug 2019 10:31:19 -0700 Subject: [PATCH 05/55] Add skeleton of stats stuff Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/BUILD | 10 ++++++++ .../clusters/redis/redis_cluster_stats.cc | 14 +++++++++++ .../clusters/redis/redis_cluster_stats.h | 24 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 source/extensions/clusters/redis/redis_cluster_stats.cc create mode 100644 source/extensions/clusters/redis/redis_cluster_stats.h diff --git a/source/extensions/clusters/redis/BUILD b/source/extensions/clusters/redis/BUILD index d6238dc9fd3d..0ceba963838e 100644 --- a/source/extensions/clusters/redis/BUILD +++ b/source/extensions/clusters/redis/BUILD @@ -66,3 +66,13 @@ envoy_cc_library( "@envoy_api//envoy/config/cluster/redis:redis_cluster_cc", ], ) + +envoy_cc_library( + name = "redis_cluster_stats_lib", + srcs = ["redis_cluster_stats.cc"], + hdrs = ["redis_cluster_stats.h"], + deps = [ + "//include/envoy/stats:stats_interface", + "//source/common/stats:symbol_table_lib", + ], +) diff --git a/source/extensions/clusters/redis/redis_cluster_stats.cc b/source/extensions/clusters/redis/redis_cluster_stats.cc new file mode 100644 index 000000000000..61ac61fb533f --- /dev/null +++ b/source/extensions/clusters/redis/redis_cluster_stats.cc @@ -0,0 +1,14 @@ +#include "redis_cluster_stats.h" + +namespace Envoy { +namespace Extensions { +namespace Clusters { +namespace Redis { + +RedisClusterStats:RedisClusterStats(Stats::Scope& scope, const std::string& prefix) + : scope_(scope), prefix_(stat_name_set_.add(prefix + "redis")) {} + +} // namespace Redis +} // namespace Clusters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/clusters/redis/redis_cluster_stats.h b/source/extensions/clusters/redis/redis_cluster_stats.h new file mode 100644 index 000000000000..22dad5d469d4 --- /dev/null +++ b/source/extensions/clusters/redis/redis_cluster_stats.h @@ -0,0 +1,24 @@ +#pragma once + +#include "envoy/stats/scope.h" + +namespace Envoy { +namespace Extensions { +namespace Clusters { +namespace Redis { + +class RedisClusterStats { +public: + RedisClusterStats(Stats::Scope& scope, const std::string& prefix); + +private: + Stats::Scope& scope_; + const Stats::StatName prefix_; + +}; +using RedisClusterStatsPtr = std::shared_ptr; + +} // namespace Redis +} // namespace Clusters +} // namespace Extensions +} // namespace Envoy From 723a8fddb5ff3815b52f2a4977388787c1eb5b76 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 14 Aug 2019 15:31:42 -0700 Subject: [PATCH 06/55] fix Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/BUILD | 10 ----- .../clusters/redis/redis_cluster_stats.cc | 14 ------- .../clusters/redis/redis_cluster_stats.h | 24 ----------- .../filters/network/common/redis/BUILD | 12 ++++++ .../network/common/redis/client_impl.cc | 3 +- .../network/common/redis/client_impl.h | 2 + .../common/redis/redis_command_stats.cc | 40 +++++++++++++++++++ .../common/redis/redis_command_stats.h | 37 +++++++++++++++++ 8 files changed, 93 insertions(+), 49 deletions(-) delete mode 100644 source/extensions/clusters/redis/redis_cluster_stats.cc delete mode 100644 source/extensions/clusters/redis/redis_cluster_stats.h create mode 100644 source/extensions/filters/network/common/redis/redis_command_stats.cc create mode 100644 source/extensions/filters/network/common/redis/redis_command_stats.h diff --git a/source/extensions/clusters/redis/BUILD b/source/extensions/clusters/redis/BUILD index 0ceba963838e..d6238dc9fd3d 100644 --- a/source/extensions/clusters/redis/BUILD +++ b/source/extensions/clusters/redis/BUILD @@ -66,13 +66,3 @@ envoy_cc_library( "@envoy_api//envoy/config/cluster/redis:redis_cluster_cc", ], ) - -envoy_cc_library( - name = "redis_cluster_stats_lib", - srcs = ["redis_cluster_stats.cc"], - hdrs = ["redis_cluster_stats.h"], - deps = [ - "//include/envoy/stats:stats_interface", - "//source/common/stats:symbol_table_lib", - ], -) diff --git a/source/extensions/clusters/redis/redis_cluster_stats.cc b/source/extensions/clusters/redis/redis_cluster_stats.cc deleted file mode 100644 index 61ac61fb533f..000000000000 --- a/source/extensions/clusters/redis/redis_cluster_stats.cc +++ /dev/null @@ -1,14 +0,0 @@ -#include "redis_cluster_stats.h" - -namespace Envoy { -namespace Extensions { -namespace Clusters { -namespace Redis { - -RedisClusterStats:RedisClusterStats(Stats::Scope& scope, const std::string& prefix) - : scope_(scope), prefix_(stat_name_set_.add(prefix + "redis")) {} - -} // namespace Redis -} // namespace Clusters -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/clusters/redis/redis_cluster_stats.h b/source/extensions/clusters/redis/redis_cluster_stats.h deleted file mode 100644 index 22dad5d469d4..000000000000 --- a/source/extensions/clusters/redis/redis_cluster_stats.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "envoy/stats/scope.h" - -namespace Envoy { -namespace Extensions { -namespace Clusters { -namespace Redis { - -class RedisClusterStats { -public: - RedisClusterStats(Stats::Scope& scope, const std::string& prefix); - -private: - Stats::Scope& scope_; - const Stats::StatName prefix_; - -}; -using RedisClusterStatsPtr = std::shared_ptr; - -} // namespace Redis -} // namespace Clusters -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index ac070361b193..6159df456181 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -55,6 +55,7 @@ envoy_cc_library( srcs = ["client_impl.cc"], hdrs = ["client_impl.h"], deps = [ + ":redis_command_stats_lib", ":client_interface", ":codec_lib", "//include/envoy/router:router_interface", @@ -79,3 +80,14 @@ envoy_cc_library( ":codec_lib", ], ) + +envoy_cc_library( + name = "redis_command_stats_lib", + srcs = ["redis_command_stats.cc"], + hdrs = ["redis_command_stats.h"], + deps = [ + ":supported_commands_lib", + "//include/envoy/stats:stats_interface", + "//source/common/stats:symbol_table_lib", + ], +) \ No newline at end of file diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index f698716141f1..0156253b995d 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -42,7 +42,8 @@ ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dis config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), - redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()) { + redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), + redis_command_stats_(std::move(std::make_shared(host->cluster().statsScope(), "foo"))) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 6cb209a16244..eea51feda603 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -16,6 +16,7 @@ #include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/client.h" +#include "extensions/filters/network/common/redis/redis_command_stats.h" namespace Envoy { namespace Extensions { @@ -132,6 +133,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr flush_timer_; RedisClusterStats redis_cluster_stats_; Envoy::TimeSource& time_source_; + const RedisCommandStatsPtr& redis_command_stats_; }; class ClientFactoryImpl : public ClientFactory { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc new file mode 100644 index 000000000000..870dba8edcae --- /dev/null +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -0,0 +1,40 @@ +#include "extensions/filters/network/common/redis/redis_command_stats.h" +#include "extensions/filters/network/common/redis/supported_commands.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + + +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix) + : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix + "commands")) { + + // Create StatName for each Redis command. Note that we don't include Auth or Ping. + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + stat_name_set_.add(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + stat_name_set_.add(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::hashMultipleSumResultCommands()) { + stat_name_set_.add(command); + } + stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); +} + +Stats::Counter& RedisCommandStats::counter(std::string name) { + return scope_.counterFromStatName(stat_name_set_.getStatName(name)); +} + +Stats::Histogram& RedisCommandStats::histogram(std::string name) { + return scope_.histogramFromStatName(stat_name_set_.getStatName(name)); +} + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h new file mode 100644 index 000000000000..b882fb7b50b8 --- /dev/null +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +#include "envoy/stats/scope.h" +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + + +class RedisCommandStats { +public: + RedisCommandStats(Stats::Scope& scope, const std::string& prefix); + + Stats::Counter& counter(std::string name); + Stats::Histogram& histogram(std::string name); + +private: + Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); + + Stats::Scope& scope_; + Stats::StatNameSet stat_name_set_; + const Stats::StatName prefix_; + +}; +using RedisCommandStatsPtr = std::shared_ptr; + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy From 2c169e2d97bc1fec1ee95e30040b8c797d6f2252 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 14 Aug 2019 15:55:48 -0700 Subject: [PATCH 07/55] record command value Signed-off-by: Nicolas Flacco --- .../extensions/filters/network/common/redis/client_impl.cc | 5 +++++ .../filters/network/common/redis/redis_command_stats.cc | 4 ---- .../filters/network/common/redis/redis_command_stats.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 0156253b995d..066eecdc0614 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -75,6 +75,11 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca pending_requests_.emplace_back(*this, callbacks); encoder_->encode(request, encoder_buffer_); + // TODO: Clean up this part + std::vector command_array = request.asArray(); + std::string command_name = command_array.front().asString(); + redis_command_stats_->counter(command_name); + // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { flushBufferAndResetTimer(); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 870dba8edcae..7d0794b4a88f 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -29,10 +29,6 @@ Stats::Counter& RedisCommandStats::counter(std::string name) { return scope_.counterFromStatName(stat_name_set_.getStatName(name)); } -Stats::Histogram& RedisCommandStats::histogram(std::string name) { - return scope_.histogramFromStatName(stat_name_set_.getStatName(name)); -} - } // namespace Redis } // namespace Common } // namespace NetworkFilters diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index b882fb7b50b8..ae6adfb5c1f3 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,7 +18,7 @@ class RedisCommandStats { RedisCommandStats(Stats::Scope& scope, const std::string& prefix); Stats::Counter& counter(std::string name); - Stats::Histogram& histogram(std::string name); + // Could add histogram timer for each command in future private: Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); From 8b05c9b44d5e3d576268b6d8e1c430edb71ec8e0 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 14 Aug 2019 16:06:05 -0700 Subject: [PATCH 08/55] fix formatting Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/BUILD | 4 ++-- .../network/common/redis/client_impl.cc | 3 ++- .../common/redis/redis_command_stats.cc | 22 +++++++++++-------- .../common/redis/redis_command_stats.h | 5 ++--- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 6159df456181..3bd94507c93d 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -55,9 +55,9 @@ envoy_cc_library( srcs = ["client_impl.cc"], hdrs = ["client_impl.h"], deps = [ - ":redis_command_stats_lib", ":client_interface", ":codec_lib", + ":redis_command_stats_lib", "//include/envoy/router:router_interface", "//include/envoy/stats:timespan", "//include/envoy/thread_local:thread_local_interface", @@ -90,4 +90,4 @@ envoy_cc_library( "//include/envoy/stats:stats_interface", "//source/common/stats:symbol_table_lib", ], -) \ No newline at end of file +) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 066eecdc0614..4bb0b00e28a6 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -43,7 +43,8 @@ ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dis connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), - redis_command_stats_(std::move(std::make_shared(host->cluster().statsScope(), "foo"))) { + redis_command_stats_( + std::move(std::make_shared(host->cluster().statsScope(), "foo"))) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 7d0794b4a88f..49a9039aef38 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -1,4 +1,5 @@ #include "extensions/filters/network/common/redis/redis_command_stats.h" + #include "extensions/filters/network/common/redis/supported_commands.h" namespace Envoy { @@ -7,19 +8,22 @@ namespace NetworkFilters { namespace Common { namespace Redis { - -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix) - : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix + "commands")) { +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix) + : scope_(scope), stat_name_set_(scope.symbolTable()), + prefix_(stat_name_set_.add(prefix + "commands")) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - stat_name_set_.add(command); + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + stat_name_set_.add(command); } - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - stat_name_set_.add(command); + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + stat_name_set_.add(command); } - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::hashMultipleSumResultCommands()) { - stat_name_set_.add(command); + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()) { + stat_name_set_.add(command); } stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index ae6adfb5c1f3..3db807b3bfd4 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -4,6 +4,7 @@ #include #include "envoy/stats/scope.h" + #include "common/stats/symbol_table_impl.h" namespace Envoy { @@ -12,7 +13,6 @@ namespace NetworkFilters { namespace Common { namespace Redis { - class RedisCommandStats { public: RedisCommandStats(Stats::Scope& scope, const std::string& prefix); @@ -22,11 +22,10 @@ class RedisCommandStats { private: Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); - + Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; - }; using RedisCommandStatsPtr = std::shared_ptr; From e74161ae1b1e24dc0cca90ba95e805ca7ae20fd4 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 14:21:52 -0700 Subject: [PATCH 09/55] update Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 4bb0b00e28a6..e460202d752b 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -43,8 +43,7 @@ ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dis connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), - redis_command_stats_( - std::move(std::make_shared(host->cluster().statsScope(), "foo"))) { + redis_command_stats_{std::make_shared(host->cluster().statsScope(), "foo")} { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); From 8336e68ddaa66808bcb9eefdf963b9307eb3f0fe Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 14:23:16 -0700 Subject: [PATCH 10/55] update Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index e460202d752b..532f0ffa4f52 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -43,7 +43,7 @@ ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dis connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), - redis_command_stats_{std::make_shared(host->cluster().statsScope(), "foo")} { + redis_command_stats_{std::move(std::make_shared(host->cluster().statsScope(), "foo"))} { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); From 359679e0937c22c5fff71bad2cb12717cc121f76 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 15:28:52 -0700 Subject: [PATCH 11/55] patched up a bit Signed-off-by: Nicolas Flacco --- .../network/common/redis/client_impl.cc | 23 ++++++++++++------- .../network/common/redis/client_impl.h | 4 ++-- .../common/redis/redis_command_stats.cc | 7 +++++- .../common/redis/redis_command_stats.h | 4 ++++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 532f0ffa4f52..dd8ee8984b37 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -25,8 +25,12 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, RedisClusterStats& redis_cluster_stats) { + // TODO: Maybe this should take a boolean not to create the stats? + auto redis_command_stats = + std::make_shared(host->cluster().statsScope(), "fml"); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), - decoder_factory, config, redis_cluster_stats)); + decoder_factory, config, redis_cluster_stats, + redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -37,13 +41,14 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisClusterStats& redis_cluster_stats) + RedisClusterStats& redis_cluster_stats, + RedisCommandStatsPtr& redis_command_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), - redis_command_stats_{std::move(std::make_shared(host->cluster().statsScope(), "foo"))} { + redis_command_stats_(redis_command_stats) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -75,10 +80,11 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca pending_requests_.emplace_back(*this, callbacks); encoder_->encode(request, encoder_buffer_); - // TODO: Clean up this part - std::vector command_array = request.asArray(); - std::string command_name = command_array.front().asString(); - redis_command_stats_->counter(command_name); + // TODO: Make this part work pretty please + // std::vector command_array = request.asArray(); + // std::string command_name = command_array.front().asString(); + // redis_command_stats_->counter(command_name); + redis_command_stats_->counter("fml"); // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { @@ -212,7 +218,8 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) : parent_(parent), callbacks_(callbacks), request_timer_(std::make_unique( - parent_.redis_cluster_stats_.upstream_rq_time_, parent_.time_source_)) { + parent_.redis_command_stats_->histogram(parent_.redis_command_stats_->upstream_rq_time_), + parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index eea51feda603..29fe07a8ff67 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -108,7 +108,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisClusterStats& redis_cluster_stats); + RedisClusterStats& redis_cluster_stats, RedisCommandStatsPtr& redis_command_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); @@ -133,7 +133,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr flush_timer_; RedisClusterStats redis_cluster_stats_; Envoy::TimeSource& time_source_; - const RedisCommandStatsPtr& redis_command_stats_; + const RedisCommandStatsPtr redis_command_stats_; }; class ClientFactoryImpl : public ClientFactory { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 49a9039aef38..5e0d8aa64ad9 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -10,7 +10,8 @@ namespace Redis { RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix) : scope_(scope), stat_name_set_(scope.symbolTable()), - prefix_(stat_name_set_.add(prefix + "commands")) { + prefix_(stat_name_set_.add(prefix + "commands")), + upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. for (const std::string& command : @@ -33,6 +34,10 @@ Stats::Counter& RedisCommandStats::counter(std::string name) { return scope_.counterFromStatName(stat_name_set_.getStatName(name)); } +Stats::Histogram& RedisCommandStats::histogram(Stats::StatName statName) { + return scope_.histogramFromStatName(statName); +} + } // namespace Redis } // namespace Common } // namespace NetworkFilters diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 3db807b3bfd4..3c3e9a7e7e91 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,6 +18,7 @@ class RedisCommandStats { RedisCommandStats(Stats::Scope& scope, const std::string& prefix); Stats::Counter& counter(std::string name); + Stats::Histogram& histogram(Stats::StatName statName); // We might want one with a string name too // Could add histogram timer for each command in future private: @@ -26,6 +27,9 @@ class RedisCommandStats { Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; + +public: + const Stats::StatName upstream_rq_time_; }; using RedisCommandStatsPtr = std::shared_ptr; From 506e724e950a47da273b36bbaa92b88ea2d0e3d8 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 15:44:15 -0700 Subject: [PATCH 12/55] fix counter Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index dd8ee8984b37..6bf00585bce6 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -84,7 +84,7 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca // std::vector command_array = request.asArray(); // std::string command_name = command_array.front().asString(); // redis_command_stats_->counter(command_name); - redis_command_stats_->counter("fml"); + redis_command_stats_->counter("fml").inc(); // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { From 054400bf5a2c1cdfbf0936230891924a07e33a60 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 15:59:30 -0700 Subject: [PATCH 13/55] Revert moving RedisClusterStats Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.cc | 6 ++---- source/extensions/clusters/redis/redis_cluster.h | 1 - .../filters/network/common/redis/client.h | 11 +---------- .../filters/network/common/redis/client_impl.cc | 14 +++++--------- .../filters/network/common/redis/client_impl.h | 7 +++---- .../filters/network/redis_proxy/conn_pool_impl.cc | 6 ++---- .../filters/network/redis_proxy/conn_pool_impl.h | 10 +++++++++- source/extensions/health_checkers/redis/redis.cc | 8 ++------ source/extensions/health_checkers/redis/redis.h | 1 - test/extensions/clusters/redis/BUILD | 1 - .../clusters/redis/redis_cluster_test.cc | 3 +-- test/extensions/filters/network/common/redis/BUILD | 1 - .../network/common/redis/client_impl_test.cc | 14 ++------------ .../network/redis_proxy/conn_pool_impl_test.cc | 3 +-- .../extensions/health_checkers/redis/redis_test.cc | 3 +-- 15 files changed, 29 insertions(+), 60 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 93665c4b2342..bf5160805944 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -34,8 +34,7 @@ RedisCluster::RedisCluster( : Config::Utility::translateClusterHosts(cluster.hosts())), local_info_(factory_context.localInfo()), random_(factory_context.random()), redis_discovery_session_(*this, redis_client_factory), lb_factory_(std::move(lb_factory)), - api_(api), redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(info()->statsScope()), - POOL_HISTOGRAM(info()->statsScope()))} { + api_(api) { const auto& locality_lb_endpoints = load_assignment_.endpoints(); for (const auto& locality_lb_endpoint : locality_lb_endpoints) { for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { @@ -241,8 +240,7 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = - client_factory_.create(host, dispatcher_, *this, parent_.redis_cluster_stats_); + client->client_ = client_factory_.create(host, dispatcher_, *this); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 55f54850fa13..d7b776353671 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -251,7 +251,6 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { envoy::api::v2::core::DataSource auth_password_datasource_; Api::Api& api_; - Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats redis_cluster_stats_; }; class RedisClusterFactory : public Upstream::ConfigurableClusterFactoryBase< diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 88b80f98669a..4e041ad33552 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -13,15 +13,6 @@ namespace Common { namespace Redis { namespace Client { -#define REDIS_CLUSTER_STATS(COUNTER, HISTOGRAM) \ - COUNTER(upstream_cx_drained) \ - COUNTER(max_upstream_unknown_connections_reached) \ - HISTOGRAM(upstream_rq_time) - -struct RedisClusterStats { - REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_HISTOGRAM_STRUCT) -}; - /** * A handle to an outbound request. */ @@ -183,7 +174,7 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisClusterStats& redis_cluster_stats) PURE; + const Config& config) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 6bf00585bce6..67f54a018278 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -23,14 +23,13 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisClusterStats& redis_cluster_stats) { + const Config& config) { // TODO: Maybe this should take a boolean not to create the stats? auto redis_command_stats = std::make_shared(host->cluster().statsScope(), "fml"); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), - decoder_factory, config, redis_cluster_stats, - redis_command_stats)); + decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -41,14 +40,12 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisClusterStats& redis_cluster_stats, RedisCommandStatsPtr& redis_command_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), - redis_cluster_stats_(redis_cluster_stats), time_source_(dispatcher.timeSource()), - redis_command_stats_(redis_command_stats) { + time_source_(dispatcher.timeSource()), redis_command_stats_(redis_command_stats) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -241,10 +238,9 @@ void ClientImpl::PendingRequest::cancel() { ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, - Event::Dispatcher& dispatcher, const Config& config, - RedisClusterStats& redis_cluster_stats) { + Event::Dispatcher& dispatcher, const Config& config) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config, redis_cluster_stats); + config); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 29fe07a8ff67..1d0526bd2d10 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -65,7 +65,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisClusterStats& redis_cluster_stats); + const Config& config); ~ClientImpl() override; @@ -108,7 +108,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisClusterStats& redis_cluster_stats, RedisCommandStatsPtr& redis_command_stats); + RedisCommandStatsPtr& redis_command_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); @@ -131,7 +131,6 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr connect_or_op_timer_; bool connected_{}; Event::TimerPtr flush_timer_; - RedisClusterStats redis_cluster_stats_; Envoy::TimeSource& time_source_; const RedisCommandStatsPtr redis_command_stats_; }; @@ -140,7 +139,7 @@ class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisClusterStats& redis_cluster_stats) override; + const Config& config) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index e2c14e9db803..44f2634aa89a 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -27,8 +27,7 @@ InstanceImpl::InstanceImpl( Api::Api& api, Stats::ScopePtr&& stats_scope) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(*stats_scope_), - POOL_HISTOGRAM(*stats_scope_))} { + POOL_COUNTER(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -197,8 +196,7 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, - parent_.redis_cluster_stats_); + client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_); client->redis_client_->addConnectionCallbacks(*client); if (!auth_password_.empty()) { // Send an AUTH command to the upstream server. diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 07decf76535b..91ea51f3e752 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -37,6 +37,14 @@ namespace ConnPool { // TODO(mattklein123): Circuit breaking // TODO(rshriram): Fault injection +#define REDIS_CLUSTER_STATS(COUNTER) \ + COUNTER(upstream_cx_drained) \ + COUNTER(max_upstream_unknown_connections_reached) + +struct RedisClusterStats { + REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT) +}; + class InstanceImpl : public Instance { public: InstanceImpl( @@ -123,7 +131,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ConfigImpl config_; Api::Api& api_; Stats::ScopePtr stats_scope_; - Common::Redis::Client::RedisClusterStats redis_cluster_stats_; + RedisClusterStats redis_cluster_stats_; }; } // namespace ConnPool diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 55c35aa1727a..9f130824639c 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -12,10 +12,7 @@ RedisHealthChecker::RedisHealthChecker( Upstream::HealthCheckEventLoggerPtr&& event_logger, Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : HealthCheckerImplBase(cluster, config, dispatcher, runtime, random, std::move(event_logger)), - client_factory_(client_factory), - key_(redis_config.key()), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(cluster.info()->statsScope()), - POOL_HISTOGRAM(cluster.info()->statsScope()))} { + client_factory_(client_factory), key_(redis_config.key()) { if (!key_.empty()) { type_ = Type::Exists; } else { @@ -54,8 +51,7 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, - parent_.redis_cluster_stats_); + client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this); client_->addConnectionCallbacks(*this); } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index f3890f194255..346b27e9b951 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -113,7 +113,6 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; Type type_; const std::string key_; - Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats redis_cluster_stats_; }; } // namespace RedisHealthChecker diff --git a/test/extensions/clusters/redis/BUILD b/test/extensions/clusters/redis/BUILD index a247c8fc2069..b990f1df93fe 100644 --- a/test/extensions/clusters/redis/BUILD +++ b/test/extensions/clusters/redis/BUILD @@ -23,7 +23,6 @@ envoy_extension_cc_test( "//source/common/upstream:upstream_lib", "//source/extensions/clusters/redis:redis_cluster", "//source/extensions/clusters/redis:redis_cluster_lb", - "//source/extensions/filters/network/common/redis:client_interface", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 06d6e023ba68..4d60412503ff 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -61,8 +61,7 @@ class RedisClusterTest : public testing::Test, // ClientFactory Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/BUILD b/test/extensions/filters/network/common/redis/BUILD index ed87543857ac..8b0f46b1600f 100644 --- a/test/extensions/filters/network/common/redis/BUILD +++ b/test/extensions/filters/network/common/redis/BUILD @@ -54,7 +54,6 @@ envoy_cc_test( "//source/common/upstream:upstream_lib", "//source/extensions/filters/network/common/redis:client_lib", "//test/mocks/network:network_mocks", - "//test/mocks/stats:stats_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", ], diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index c177f9f06969..064c740dacc6 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -43,10 +43,6 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF return Common::Redis::DecoderPtr{decoder_}; } - RedisClientImplTest() - : redis_cluster_stats_(RedisClusterStats{ - REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats_), POOL_HISTOGRAM(fake_stats_))}) {} - ~RedisClientImplTest() override { client_.reset(); @@ -81,7 +77,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, noDelay(true)); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_, redis_cluster_stats_); + *config_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -118,8 +114,6 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Network::ReadFilterSharedPtr upstream_read_filter_; std::unique_ptr config_; ClientPtr client_; - Stats::IsolatedStoreImpl fake_stats_; - RedisClusterStats redis_cluster_stats_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { @@ -882,11 +876,7 @@ TEST(RedisClientFactoryImplTest, Basic) { EXPECT_CALL(*host, createConnection_(_, _)).WillOnce(Return(conn_info)); NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); - Stats::IsolatedStoreImpl fake_stats; - RedisClusterStats redis_cluster_stats = - RedisClusterStats{REDIS_CLUSTER_STATS(POOL_COUNTER(fake_stats), POOL_HISTOGRAM(fake_stats))}; - - ClientPtr client = factory.create(host, dispatcher, config, redis_cluster_stats); + ClientPtr client = factory.create(host, dispatcher, config); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 4d24474c1182..ee16cfaa5015 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -159,8 +159,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Common::Redis::Client::Config&, - Common::Redis::Client::RedisClusterStats&) override { + const Common::Redis::Client::Config&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index d01bfd7ba66c..d5741a158eb4 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -126,8 +126,7 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::Client::RedisClusterStats&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From e5770f18c6927c75f9ffad94c04018f1d5511db7 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 16:02:32 -0700 Subject: [PATCH 14/55] disable timer Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 67f54a018278..a95ed3baab91 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -81,7 +81,7 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca // std::vector command_array = request.asArray(); // std::string command_name = command_array.front().asString(); // redis_command_stats_->counter(command_name); - redis_command_stats_->counter("fml").inc(); + // redis_command_stats_->counter("fml").inc(); // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { From a110c26fa8a5afd7369cf8c095c95007773d80e4 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 16:48:57 -0700 Subject: [PATCH 15/55] Add config setting to enable stats Signed-off-by: Nicolas Flacco --- .../network/redis_proxy/v2/redis_proxy.proto | 4 ++ .../extensions/clusters/redis/redis_cluster.h | 3 +- .../filters/network/common/redis/client.h | 5 +++ .../network/common/redis/client_impl.cc | 19 +++++----- .../network/common/redis/client_impl.h | 2 + .../common/redis/redis_command_stats.cc | 37 ++++++++++--------- .../common/redis/redis_command_stats.h | 4 +- .../extensions/health_checkers/redis/redis.h | 1 + .../network/common/redis/client_impl_test.cc | 2 + .../health_checkers/redis/redis_test.cc | 1 + 10 files changed, 49 insertions(+), 29 deletions(-) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 9f714dd66cd0..4b07c0d91ca2 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -90,6 +90,10 @@ message RedisProxy { // this limit, then redirection will fail and the original redirection error will be passed // downstream unchanged. This limit defaults to 100. google.protobuf.UInt32Value max_upstream_unknown_connections = 6; + + // Enable per-command statistics per upstream cluster, in addition to the filter level aggregate + // count. + bool enable_command_stats = 7; } // Network settings for the connection pool to the upstream clusters. diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index d7b776353671..7b841d4d08d1 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -212,7 +212,8 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { uint32_t maxBufferSizeBeforeFlush() const override { return 0; } std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } - + bool enableCommandStats() const override { return false; } + // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; void onFailure() override; diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 4e041ad33552..c57cd4daf49e 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -157,6 +157,11 @@ class Config { * minimize the need for a large maxUpstreamUnknownConnections() value. */ virtual uint32_t maxUpstreamUnknownConnections() const PURE; + + /** + * @return when enabled, upstream cluster per-command statistics will be recorded. + */ + virtual bool enableCommandStats() const PURE; }; /** diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index a95ed3baab91..cb27d430cb0c 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -19,15 +19,14 @@ ConfigImpl::ConfigImpl( 3)), // Default timeout is 3ms. If max_buffer_size_before_flush is zero, this is not used // as the buffer is flushed on each request immediately. max_upstream_unknown_connections_( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)) {} + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)), + enable_command_stats_(config.enable_command_stats()) {} ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - - // TODO: Maybe this should take a boolean not to create the stats? auto redis_command_stats = - std::make_shared(host->cluster().statsScope(), "fml"); + std::make_shared(host->cluster().statsScope(), "fml", config.enableCommandStats()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; @@ -77,11 +76,13 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca pending_requests_.emplace_back(*this, callbacks); encoder_->encode(request, encoder_buffer_); - // TODO: Make this part work pretty please - // std::vector command_array = request.asArray(); - // std::string command_name = command_array.front().asString(); - // redis_command_stats_->counter(command_name); - // redis_command_stats_->counter("fml").inc(); + if (config_.enableCommandStats()) { + // TODO: This doesn't work. Need to remember where to get the command string from! + // std::vector command_array = request.asArray(); + // std::string command_name = command_array.front().asString(); + // redis_command_stats_->counter(command_name); + redis_command_stats_->counter("fml").inc(); + } // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 1d0526bd2d10..c2a80f2c836f 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -51,6 +51,7 @@ class ConfigImpl : public Config { uint32_t maxUpstreamUnknownConnections() const override { return max_upstream_unknown_connections_; } + bool enableCommandStats() const override { return enable_command_stats_; } private: const std::chrono::milliseconds op_timeout_; @@ -59,6 +60,7 @@ class ConfigImpl : public Config { const uint32_t max_buffer_size_before_flush_; const std::chrono::milliseconds buffer_flush_timeout_; const uint32_t max_upstream_unknown_connections_; + const bool enable_command_stats_; }; class ClientImpl : public Client, public DecoderCallbacks, public Network::ConnectionCallbacks { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 5e0d8aa64ad9..d59388291d17 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,26 +8,29 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix) +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enableCommandCounts) : scope_(scope), stat_name_set_(scope.symbolTable()), - prefix_(stat_name_set_.add(prefix + "commands")), + prefix_(stat_name_set_.add(prefix)), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { - - // Create StatName for each Redis command. Note that we don't include Auth or Ping. - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - stat_name_set_.add(command); - } - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - stat_name_set_.add(command); - } - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: - hashMultipleSumResultCommands()) { - stat_name_set_.add(command); + + // Note: Even if enableCommandCounts is disabled, we track the upstrea_rq_time. + if (enableCommandCounts) { + // Create StatName for each Redis command. Note that we don't include Auth or Ping. + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + stat_name_set_.add(command); + } + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + stat_name_set_.add(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()) { + stat_name_set_.add(command); + } + stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } - stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } Stats::Counter& RedisCommandStats::counter(std::string name) { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 3c3e9a7e7e91..8189a5c4c5fc 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -15,10 +15,10 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix); + RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enableCommandCounts); Stats::Counter& counter(std::string name); - Stats::Histogram& histogram(Stats::StatName statName); // We might want one with a string name too + Stats::Histogram& histogram(Stats::StatName statName); // This is only used by upstream_rq_time_ currently, so no need to do a dynamic lookup // Could add histogram timer for each command in future private: diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 346b27e9b951..cccc616a4dc9 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -78,6 +78,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 064c740dacc6..528c7981e26d 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -158,6 +158,7 @@ class ConfigBufferSizeGTSingleRequest : public Config { return std::chrono::milliseconds(1); } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } }; TEST_F(RedisClientImplTest, BatchWithTimerFiring) { @@ -470,6 +471,7 @@ class ConfigOutlierDisabled : public Config { return std::chrono::milliseconds(0); } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index d5741a158eb4..14839048869a 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -168,6 +168,7 @@ class RedisHealthCheckerTest EXPECT_EQ(session->maxBufferSizeBeforeFlush(), 0); EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); + EXPECT_FALSE(session->enableCommandStats()); session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 4f7285609e3fcf7ef7fd944834eb8dfd10a0d3fc Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 16:59:44 -0700 Subject: [PATCH 16/55] mend Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.h | 2 +- .../filters/network/common/redis/client_impl.cc | 4 ++-- .../network/common/redis/redis_command_stats.cc | 16 ++++++++-------- .../network/common/redis/redis_command_stats.h | 4 +++- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 7b841d4d08d1..6e6ad52e3cc6 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -213,7 +213,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - + // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; void onFailure() override; diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index cb27d430cb0c..d9887e44d66d 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -25,8 +25,8 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - auto redis_command_stats = - std::make_shared(host->cluster().statsScope(), "fml", config.enableCommandStats()); + auto redis_command_stats = std::make_shared( + host->cluster().statsScope(), "fml", config.enableCommandStats()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index d59388291d17..1a466ce2e13c 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,24 +8,24 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enableCommandCounts) - : scope_(scope), stat_name_set_(scope.symbolTable()), - prefix_(stat_name_set_.add(prefix)), +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, + bool enableCommandCounts) + : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { - - // Note: Even if enableCommandCounts is disabled, we track the upstrea_rq_time. + + // Note: Even if this is disabled, we track the upstream_rq_time. if (enableCommandCounts) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { stat_name_set_.add(command); } for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { stat_name_set_.add(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: - hashMultipleSumResultCommands()) { + hashMultipleSumResultCommands()) { stat_name_set_.add(command); } stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 8189a5c4c5fc..66bb7997beef 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,7 +18,9 @@ class RedisCommandStats { RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enableCommandCounts); Stats::Counter& counter(std::string name); - Stats::Histogram& histogram(Stats::StatName statName); // This is only used by upstream_rq_time_ currently, so no need to do a dynamic lookup + Stats::Histogram& + histogram(Stats::StatName statName); // This is only used by upstream_rq_time_ currently, so no + // need to do a dynamic lookup // Could add histogram timer for each command in future private: From 7af7c7c4e950d676bf71edc98cbaa192b724ed18 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 17:05:01 -0700 Subject: [PATCH 17/55] fix Signed-off-by: Nicolas Flacco --- source/extensions/health_checkers/redis/redis.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index cccc616a4dc9..d52974ccce60 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -78,7 +78,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } - bool enableCommandStats() const override { return false } + bool enableCommandStats() const override { return false; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; From e06eb57e980abbbbeb72f984fad6df79bc245994 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 17:18:36 -0700 Subject: [PATCH 18/55] try to fix Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index d9887e44d66d..5be1481468ad 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -25,8 +25,8 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - auto redis_command_stats = std::make_shared( - host->cluster().statsScope(), "fml", config.enableCommandStats()); + auto redis_command_stats = + std::make_shared(host->cluster().statsScope(), "fml", true); // TODO: fix? std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; From 04314f5b4d749bd0cb5ced8b96950c33fef5d19d Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 15 Aug 2019 17:26:14 -0700 Subject: [PATCH 19/55] revert Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 5be1481468ad..d9887e44d66d 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -25,8 +25,8 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - auto redis_command_stats = - std::make_shared(host->cluster().statsScope(), "fml", true); // TODO: fix? + auto redis_command_stats = std::make_shared( + host->cluster().statsScope(), "fml", config.enableCommandStats()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; From 3e8636e6706bfb73930b4d312905db5e7c7bd056 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 16 Aug 2019 12:18:50 -0700 Subject: [PATCH 20/55] fix stats Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/client_impl.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index d9887e44d66d..871211c5487f 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -77,11 +77,12 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca encoder_->encode(request, encoder_buffer_); if (config_.enableCommandStats()) { - // TODO: This doesn't work. Need to remember where to get the command string from! - // std::vector command_array = request.asArray(); - // std::string command_name = command_array.front().asString(); - // redis_command_stats_->counter(command_name); - redis_command_stats_->counter("fml").inc(); + // Get command from RespValue + if (request.type() == RespType::Array) { + redis_command_stats_->counter(request.asArray().front().toString()); + } else { + redis_command_stats_->counter(request.toString()); + } } // If buffer is full, flush. If the buffer was empty before the request, start the timer. From 6c5a8ca3d2c448b68d9e0d5a861e1514f0fff021 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 16 Aug 2019 13:41:36 -0700 Subject: [PATCH 21/55] TODO: Remove prefix? Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/client_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 871211c5487f..891520461ae3 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -79,9 +79,9 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca if (config_.enableCommandStats()) { // Get command from RespValue if (request.type() == RespType::Array) { - redis_command_stats_->counter(request.asArray().front().toString()); + redis_command_stats_->counter("fml." + request.asArray().front().asString()).inc(); } else { - redis_command_stats_->counter(request.toString()); + redis_command_stats_->counter("fml." + request.asString()).inc(); } } From 89c1b2697c07e1bc2ee3c90e5f8c622bae2e5fdf Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 16 Aug 2019 14:08:32 -0700 Subject: [PATCH 22/55] improve stat prefix? Signed-off-by: Nicolas Flacco --- .../network/common/redis/redis_command_stats.cc | 15 +++++++++++++-- .../network/common/redis/redis_command_stats.h | 2 +- .../network/redis_proxy/command_splitter_impl.cc | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 1a466ce2e13c..3e0d472f911e 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -33,12 +33,23 @@ RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& pre } } +Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatName name) { + Stats::StatNameVec names_with_prefix; + names_with_prefix.reserve(2); + names_with_prefix.push_back(prefix_); + names_with_prefix.insert(names_with_prefix.end(), name); + return scope_.symbolTable().join(names_with_prefix); +} + Stats::Counter& RedisCommandStats::counter(std::string name) { - return scope_.counterFromStatName(stat_name_set_.getStatName(name)); + Stats::StatName statName = stat_name_set_.getStatName(name); + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(statName); + return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); } Stats::Histogram& RedisCommandStats::histogram(Stats::StatName statName) { - return scope_.histogramFromStatName(statName); + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(statName); + return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); } } // namespace Redis diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 66bb7997beef..e44477ba6bf4 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -24,7 +24,7 @@ class RedisCommandStats { // Could add histogram timer for each command in future private: - Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); + Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc index c148825a7909..158e60cf20ef 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc @@ -620,7 +620,7 @@ SplitRequestPtr InstanceImpl::makeRequest(Common::Redis::RespValuePtr&& request, } auto handler = handler_lookup_table_.find(to_lower_string.c_str()); - if (handler == nullptr) { + if (handler == nullptr) { stats_.unsupported_command_.inc(); callbacks.onResponse(Utility::makeError( fmt::format("unsupported command '{}'", request->asArray()[0].asString()))); From b4e5944e731d2ca0e8e66f8289da301e972d3a2c Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 16 Aug 2019 14:11:00 -0700 Subject: [PATCH 23/55] fix ws/format Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/redis_command_stats.h | 2 +- .../filters/network/redis_proxy/command_splitter_impl.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index e44477ba6bf4..72d6049aaad2 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -24,7 +24,7 @@ class RedisCommandStats { // Could add histogram timer for each command in future private: - Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); + Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc index 158e60cf20ef..c148825a7909 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc @@ -620,7 +620,7 @@ SplitRequestPtr InstanceImpl::makeRequest(Common::Redis::RespValuePtr&& request, } auto handler = handler_lookup_table_.find(to_lower_string.c_str()); - if (handler == nullptr) { + if (handler == nullptr) { stats_.unsupported_command_.inc(); callbacks.onResponse(Utility::makeError( fmt::format("unsupported command '{}'", request->asArray()[0].asString()))); From ab8047b004a9e8e175c74647f925407d56181706 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 16 Aug 2019 14:24:53 -0700 Subject: [PATCH 24/55] fix prefixing Signed-off-by: Nicolas Flacco --- .../extensions/filters/network/common/redis/client_impl.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 891520461ae3..7a6d4ee69ee9 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -26,7 +26,7 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { auto redis_command_stats = std::make_shared( - host->cluster().statsScope(), "fml", config.enableCommandStats()); + host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; @@ -79,9 +79,9 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca if (config_.enableCommandStats()) { // Get command from RespValue if (request.type() == RespType::Array) { - redis_command_stats_->counter("fml." + request.asArray().front().asString()).inc(); + redis_command_stats_->counter(request.asArray().front().asString()).inc(); } else { - redis_command_stats_->counter("fml." + request.asString()).inc(); + redis_command_stats_->counter(request.asString()).inc(); } } From 2441b00684443dfdc0a656dfb7234010ddb266c3 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 19 Aug 2019 17:20:46 -0700 Subject: [PATCH 25/55] update to get command latency Signed-off-by: Nicolas Flacco --- .../network/redis_proxy/v2/redis_proxy.proto | 10 ++- .../extensions/clusters/redis/redis_cluster.h | 1 + .../filters/network/common/redis/BUILD | 2 + .../filters/network/common/redis/client.h | 6 ++ .../network/common/redis/client_impl.cc | 39 +++++---- .../network/common/redis/client_impl.h | 8 +- .../common/redis/redis_command_stats.cc | 79 ++++++++++++++++--- .../common/redis/redis_command_stats.h | 22 ++++-- .../filters/network/redis_proxy/config.cc | 2 +- .../extensions/health_checkers/redis/redis.h | 1 + .../network/common/redis/client_impl_test.cc | 2 + .../health_checkers/redis/redis_test.cc | 2 + 12 files changed, 136 insertions(+), 38 deletions(-) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 4b07c0d91ca2..4eb32fa85cf2 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -94,6 +94,10 @@ message RedisProxy { // Enable per-command statistics per upstream cluster, in addition to the filter level aggregate // count. bool enable_command_stats = 7; + + // Indicates that latency stat should be computed in microseconds. By default it is computed in + // milliseconds. + bool latency_in_micros = 8; } // Network settings for the connection pool to the upstream clusters. @@ -101,7 +105,11 @@ message RedisProxy { // Indicates that latency stat should be computed in microseconds. By default it is computed in // milliseconds. - bool latency_in_micros = 4; + // + // This field is deprecated. Use a :ref:`latency_in_micros + // route` + // instead. + bool latency_in_micros = 4 [deprecated = true]; message PrefixRoutes { message Route { diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 6e6ad52e3cc6..4691c06ac9bb 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -213,6 +213,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool latencyInMicros() const override { return false; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 3bd94507c93d..52ccdf23b139 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -86,8 +86,10 @@ envoy_cc_library( srcs = ["redis_command_stats.cc"], hdrs = ["redis_command_stats.h"], deps = [ + ":codec_interface", ":supported_commands_lib", "//include/envoy/stats:stats_interface", + "//include/envoy/stats:timespan", "//source/common/stats:symbol_table_lib", ], ) diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index c57cd4daf49e..39d99982ccf3 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -162,6 +162,12 @@ class Config { * @return when enabled, upstream cluster per-command statistics will be recorded. */ virtual bool enableCommandStats() const PURE; + + /** + * @return when enabled, upstream cluster per-command latency statistics will be recorded in + * microseconds. + */ + virtual bool latencyInMicros() const PURE; }; /** diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 7a6d4ee69ee9..4289d1d2b8b2 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -20,13 +20,15 @@ ConfigImpl::ConfigImpl( // as the buffer is flushed on each request immediately. max_upstream_unknown_connections_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)), - enable_command_stats_(config.enable_command_stats()) {} + enable_command_stats_(config.enable_command_stats()), + latency_in_micros_(config.latency_in_micros()) {} ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - auto redis_command_stats = std::make_shared( - host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); + auto redis_command_stats = + std::make_shared(host->cluster().statsScope(), "upstream_commands", + config.enableCommandStats(), config.latencyInMicros()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; @@ -73,16 +75,12 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca const bool empty_buffer = encoder_buffer_.length() == 0; - pending_requests_.emplace_back(*this, callbacks); + std::string command = redis_command_stats_->getCommandFromRequest(request); + pending_requests_.emplace_back(*this, callbacks, command); encoder_->encode(request, encoder_buffer_); if (config_.enableCommandStats()) { - // Get command from RespValue - if (request.type() == RespType::Array) { - redis_command_stats_->counter(request.asArray().front().asString()).inc(); - } else { - redis_command_stats_->counter(request.asString()).inc(); - } + redis_command_stats_->updateStatsTotal(command); } // If buffer is full, flush. If the buffer was empty before the request, start the timer. @@ -174,7 +172,14 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ASSERT(!pending_requests_.empty()); PendingRequest& request = pending_requests_.front(); const bool canceled = request.canceled_; - request.request_timer_->complete(); + + if (redis_command_stats_->enabled()) { + bool success = !canceled && (value->type() != Common::Redis::RespType::Error); + redis_command_stats_->updateStats(success, request.command_); + request.aggregate_request_timer_->complete(); + request.command_request_timer_->complete(); + } + PoolCallbacks& callbacks = request.callbacks_; // We need to ensure the request is popped before calling the callback, since the callback might @@ -214,11 +219,13 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { putOutlierEvent(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS); } -ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) - : parent_(parent), callbacks_(callbacks), - request_timer_(std::make_unique( - parent_.redis_command_stats_->histogram(parent_.redis_command_stats_->upstream_rq_time_), - parent_.time_source_)) { +ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, + std::string command) + : parent_(parent), callbacks_(callbacks), command_{command}, + aggregate_request_timer_(parent_.redis_command_stats_->createTimer( + parent_.redis_command_stats_->upstream_rq_time_, parent_.time_source_)), + command_request_timer_( + parent_.redis_command_stats_->createTimer(command_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index c2a80f2c836f..b6a4c434a543 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -52,6 +52,7 @@ class ConfigImpl : public Config { return max_upstream_unknown_connections_; } bool enableCommandStats() const override { return enable_command_stats_; } + bool latencyInMicros() const override { return latency_in_micros_; } private: const std::chrono::milliseconds op_timeout_; @@ -61,6 +62,7 @@ class ConfigImpl : public Config { const std::chrono::milliseconds buffer_flush_timeout_; const uint32_t max_upstream_unknown_connections_; const bool enable_command_stats_; + const bool latency_in_micros_; }; class ClientImpl : public Client, public DecoderCallbacks, public Network::ConnectionCallbacks { @@ -96,7 +98,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne }; struct PendingRequest : public PoolRequest { - PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks); + PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, std::string command); ~PendingRequest() override; // PoolRequest @@ -104,8 +106,10 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl& parent_; PoolCallbacks& callbacks_; + std::string command_; bool canceled_{}; - Stats::TimespanPtr request_timer_; + Stats::CompletableTimespanPtr aggregate_request_timer_; + Stats::CompletableTimespanPtr command_request_timer_; }; ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 3e0d472f911e..1eed04cbd89f 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,31 +8,38 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, - bool enableCommandCounts) +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled, + bool latency_in_micros) : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), + latency_in_micros_(latency_in_micros), enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { - // Note: Even if this is disabled, we track the upstream_rq_time. - if (enableCommandCounts) { + if (enabled_) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - stat_name_set_.add(command); + createStats(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - stat_name_set_.add(command); + createStats(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: hashMultipleSumResultCommands()) { - stat_name_set_.add(command); + createStats(command); } - stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - stat_name_set_.add(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); + createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } } +void RedisCommandStats::createStats(std::string name) { + stat_name_set_.add(name + ".total"); + stat_name_set_.add(name + ".success"); + stat_name_set_.add(name + ".error"); + stat_name_set_.add(name + ".latency"); +} + Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatName name) { Stats::StatNameVec names_with_prefix; names_with_prefix.reserve(2); @@ -42,16 +49,62 @@ Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatNam } Stats::Counter& RedisCommandStats::counter(std::string name) { - Stats::StatName statName = stat_name_set_.getStatName(name); - const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(statName); + Stats::StatName stat_name = stat_name_set_.getStatName(name); + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(stat_name); return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); } -Stats::Histogram& RedisCommandStats::histogram(Stats::StatName statName) { - const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(statName); +Stats::Histogram& RedisCommandStats::histogram(std::string name) { + Stats::StatName stat_name = stat_name_set_.getStatName(name); + return histogram(stat_name); +} + +Stats::Histogram& RedisCommandStats::histogram(Stats::StatName stat_name) { + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(stat_name); return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); } +Stats::CompletableTimespanPtr RedisCommandStats::createTimer(std::string name, + Envoy::TimeSource& time_source) { + Stats::StatName stat_name = stat_name_set_.getStatName(name); + return createTimer(stat_name, time_source); +} + +Stats::CompletableTimespanPtr RedisCommandStats::createTimer(Stats::StatName stat_name, + Envoy::TimeSource& time_source) { + if (latency_in_micros_) { + return std::make_unique>( + histogram(stat_name), time_source); + } else { + return std::make_unique>( + histogram(stat_name), time_source); + } +} + +std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { + // Get command from RespValue + switch (request.type()) { + case RespType::Array: + return getCommandFromRequest(request.asArray().front()); + case RespType::Integer: + return std::to_string(request.asInteger()); + case RespType::Null: + return "null"; + default: + return request.asString(); + } +} + +void RedisCommandStats::updateStatsTotal(std::string command) { counter(command + ".total").inc(); } + +void RedisCommandStats::updateStats(const bool success, std::string command) { + if (success) { + counter(command + ".success").inc(); + } else { + counter(command + ".error").inc(); + } +} + } // namespace Redis } // namespace Common } // namespace NetworkFilters diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 72d6049aaad2..589c8f90eb52 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -4,9 +4,12 @@ #include #include "envoy/stats/scope.h" +#include "envoy/stats/timespan.h" #include "common/stats/symbol_table_impl.h" +#include "extensions/filters/network/common/redis/codec.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -15,20 +18,29 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enableCommandCounts); + RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled, + bool latency_in_micros); Stats::Counter& counter(std::string name); - Stats::Histogram& - histogram(Stats::StatName statName); // This is only used by upstream_rq_time_ currently, so no - // need to do a dynamic lookup - // Could add histogram timer for each command in future + Stats::Histogram& histogram(std::string name); + Stats::Histogram& histogram(Stats::StatName stat_name); + Stats::CompletableTimespanPtr createTimer(std::string name, Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createTimer(Stats::StatName stat_name, + Envoy::TimeSource& time_source); + std::string getCommandFromRequest(const RespValue& request); + void updateStatsTotal(std::string command); + void updateStats(const bool success, std::string command); + bool enabled() { return enabled_; } private: + void createStats(std::string name); Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; + bool latency_in_micros_; + bool enabled_; public: const Stats::StatName upstream_rq_time_; diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 5e3e4018260d..769e542bec79 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -74,7 +74,7 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP std::shared_ptr splitter = std::make_shared( std::move(router), context.scope(), filter_config->stat_prefix_, context.timeSource(), - proto_config.latency_in_micros()); + proto_config.settings().latency_in_micros()); return [splitter, filter_config](Network::FilterManager& filter_manager) -> void { Common::Redis::DecoderFactoryImpl factory; filter_manager.addReadFilter(std::make_shared( diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index d52974ccce60..6bfd88a9fb33 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -79,6 +79,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool latencyInMicros() const override { return false; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 528c7981e26d..9f635bbe2f3e 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -159,6 +159,7 @@ class ConfigBufferSizeGTSingleRequest : public Config { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool latencyInMicros() const override { return false; } }; TEST_F(RedisClientImplTest, BatchWithTimerFiring) { @@ -472,6 +473,7 @@ class ConfigOutlierDisabled : public Config { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool latencyInMicros() const override { return false; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 14839048869a..eeafef0b07a6 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -169,6 +169,8 @@ class RedisHealthCheckerTest EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); EXPECT_FALSE(session->enableCommandStats()); + EXPECT_FALSE(session->latencyInMicros()); + session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 90afb6f00028b5ab4a93267e3acba5f5ca22ed77 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 19 Aug 2019 17:26:38 -0700 Subject: [PATCH 26/55] update per henry Signed-off-by: Nicolas Flacco --- .../extensions/filters/network/common/redis/client_impl.cc | 5 +++-- source/extensions/filters/network/common/redis/client_impl.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 4289d1d2b8b2..2393427c32b7 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -30,7 +30,8 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche std::make_shared(host->cluster().statsScope(), "upstream_commands", config.enableCommandStats(), config.latencyInMicros()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), - decoder_factory, config, redis_command_stats)); + decoder_factory, config, + std::move(redis_command_stats))); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -41,7 +42,7 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsPtr& redis_command_stats) + RedisCommandStatsPtr&& redis_command_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index b6a4c434a543..0d22726757ed 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -114,7 +114,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsPtr& redis_command_stats); + RedisCommandStatsPtr&& redis_command_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); From c9450582a7fed3d14536d386152414570d2744ed Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 19 Aug 2019 22:12:59 -0700 Subject: [PATCH 27/55] remove uneeded millis option, make latency timer have latency suffix Signed-off-by: Nicolas Flacco --- .../network/redis_proxy/v2/redis_proxy.proto | 10 +------ .../network_filters/redis_proxy_filter.rst | 4 +-- .../extensions/clusters/redis/redis_cluster.h | 1 - .../filters/network/common/redis/client.h | 6 ---- .../network/common/redis/client_impl.cc | 14 ++++------ .../network/common/redis/client_impl.h | 2 -- .../common/redis/redis_command_stats.cc | 28 ++++++++----------- .../common/redis/redis_command_stats.h | 11 ++++---- .../filters/network/redis_proxy/config.cc | 2 +- .../extensions/health_checkers/redis/redis.h | 1 - .../network/common/redis/client_impl_test.cc | 2 -- .../health_checkers/redis/redis_test.cc | 1 - 12 files changed, 25 insertions(+), 57 deletions(-) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 063fde884cee..47fbb8515c0b 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -95,10 +95,6 @@ message RedisProxy { // count. bool enable_command_stats = 8; - // Indicates that latency stat should be computed in microseconds. By default it is computed in - // milliseconds. - bool latency_in_micros = 9; - // ReadPolicy controls how Envoy routes read commands to Redis nodes. This is currently // supported for Redis Cluster. All ReadPolicy settings except MASTER may return stale data // because replication is asynchronous and requires some delay. You need to ensure that your @@ -128,11 +124,7 @@ message RedisProxy { // Indicates that latency stat should be computed in microseconds. By default it is computed in // milliseconds. - // - // This field is deprecated. Use a :ref:`latency_in_micros - // route` - // instead. - bool latency_in_micros = 4 [deprecated = true]; + bool latency_in_micros = 4; message PrefixRoutes { message Route { diff --git a/docs/root/configuration/network_filters/redis_proxy_filter.rst b/docs/root/configuration/network_filters/redis_proxy_filter.rst index 1dd12ee50ad5..1a94abb013b1 100644 --- a/docs/root/configuration/network_filters/redis_proxy_filter.rst +++ b/docs/root/configuration/network_filters/redis_proxy_filter.rst @@ -47,9 +47,7 @@ The Redis filter will gather statistics for the command splitter in the Per command statistics ---------------------- -The Redis filter will gather statistics for commands in the -*redis..command..* namespace. By default latency stats are in milliseconds and can be -changed to microseconds by setting the configuration parameter :ref:`latency_in_micros ` to true. +The Redis filter will gather statistics for commands in the *redis..upstream_commands..* namespace. .. csv-table:: :header: Name, Type, Description diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index c4d62f04a31a..9723c0d8082a 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -215,7 +215,6 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - bool latencyInMicros() const override { return false; } // This is effectively not in used for making the "Cluster Slots" calls. // since we call cluster slots on both the master and slaves, ANY is more appropriate here. Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 405383264225..1432eb43af8d 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -168,12 +168,6 @@ class Config { */ virtual bool enableCommandStats() const PURE; - /** - * @return when enabled, upstream cluster per-command latency statistics will be recorded in - * microseconds. - */ - virtual bool latencyInMicros() const PURE; - /** * @return the read policy the proxy should use. */ diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 85867a613c42..00438320f1db 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -20,8 +20,7 @@ ConfigImpl::ConfigImpl( // as the buffer is flushed on each request immediately. max_upstream_unknown_connections_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)), - enable_command_stats_(config.enable_command_stats()), - latency_in_micros_(config.latency_in_micros()) { + enable_command_stats_(config.enable_command_stats()) { switch (config.read_policy()) { case envoy::config::filter::network::redis_proxy::v2:: RedisProxy_ConnPoolSettings_ReadPolicy_MASTER: @@ -51,9 +50,8 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) { - auto redis_command_stats = - std::make_shared(host->cluster().statsScope(), "upstream_commands", - config.enableCommandStats(), config.latencyInMicros()); + auto redis_command_stats = std::make_shared( + host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config, std::move(redis_command_stats))); @@ -248,10 +246,10 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, std::string command) : parent_(parent), callbacks_(callbacks), command_{command}, - aggregate_request_timer_(parent_.redis_command_stats_->createTimer( - parent_.redis_command_stats_->upstream_rq_time_, parent_.time_source_)), + aggregate_request_timer_( + parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)), command_request_timer_( - parent_.redis_command_stats_->createTimer(command_, parent_.time_source_)) { + parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 4f337ad99a26..4f1394763500 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -52,7 +52,6 @@ class ConfigImpl : public Config { return max_upstream_unknown_connections_; } bool enableCommandStats() const override { return enable_command_stats_; } - bool latencyInMicros() const override { return latency_in_micros_; } ReadPolicy readPolicy() const override { return read_policy_; } private: @@ -63,7 +62,6 @@ class ConfigImpl : public Config { const std::chrono::milliseconds buffer_flush_timeout_; const uint32_t max_upstream_unknown_connections_; const bool enable_command_stats_; - const bool latency_in_micros_; ReadPolicy read_policy_; }; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 1eed04cbd89f..bddad0e81259 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,11 +8,9 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled, - bool latency_in_micros) +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - latency_in_micros_(latency_in_micros), enabled_(enabled), - upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { + enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { // Note: Even if this is disabled, we track the upstream_rq_time. if (enabled_) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. @@ -64,21 +62,17 @@ Stats::Histogram& RedisCommandStats::histogram(Stats::StatName stat_name) { return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); } -Stats::CompletableTimespanPtr RedisCommandStats::createTimer(std::string name, - Envoy::TimeSource& time_source) { - Stats::StatName stat_name = stat_name_set_.getStatName(name); - return createTimer(stat_name, time_source); +Stats::CompletableTimespanPtr +RedisCommandStats::createCommandTimer(std::string name, Envoy::TimeSource& time_source) { + Stats::StatName stat_name = stat_name_set_.getStatName(name + latency_suffix_); + return std::make_unique>(histogram(stat_name), + time_source); } -Stats::CompletableTimespanPtr RedisCommandStats::createTimer(Stats::StatName stat_name, - Envoy::TimeSource& time_source) { - if (latency_in_micros_) { - return std::make_unique>( - histogram(stat_name), time_source); - } else { - return std::make_unique>( - histogram(stat_name), time_source); - } +Stats::CompletableTimespanPtr +RedisCommandStats::createAggregateTimer(Envoy::TimeSource& time_source) { + return std::make_unique>( + histogram(upstream_rq_time_), time_source); } std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 589c8f90eb52..053b0d295c13 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,15 +18,14 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled, - bool latency_in_micros); + RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled); Stats::Counter& counter(std::string name); Stats::Histogram& histogram(std::string name); Stats::Histogram& histogram(Stats::StatName stat_name); - Stats::CompletableTimespanPtr createTimer(std::string name, Envoy::TimeSource& time_source); - Stats::CompletableTimespanPtr createTimer(Stats::StatName stat_name, - Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createCommandTimer(std::string name, + Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createAggregateTimer(Envoy::TimeSource& time_source); std::string getCommandFromRequest(const RespValue& request); void updateStatsTotal(std::string command); void updateStats(const bool success, std::string command); @@ -39,8 +38,8 @@ class RedisCommandStats { Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; - bool latency_in_micros_; bool enabled_; + const std::string latency_suffix_ = ".latency"; public: const Stats::StatName upstream_rq_time_; diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 769e542bec79..5e3e4018260d 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -74,7 +74,7 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP std::shared_ptr splitter = std::make_shared( std::move(router), context.scope(), filter_config->stat_prefix_, context.timeSource(), - proto_config.settings().latency_in_micros()); + proto_config.latency_in_micros()); return [splitter, filter_config](Network::FilterManager& filter_manager) -> void { Common::Redis::DecoderFactoryImpl factory; filter_manager.addReadFilter(std::make_shared( diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 46fdf8cf7739..d5d33542003f 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -82,7 +82,6 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - bool latencyInMicros() const override { return false; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index a578fe09613b..a0cd89af9e54 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -156,7 +156,6 @@ class ConfigBufferSizeGTSingleRequest : public Config { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - bool latencyInMicros() const override { return false; } ReadPolicy readPolicy() const override { return ReadPolicy::Master; } }; @@ -472,7 +471,6 @@ class ConfigOutlierDisabled : public Config { ReadPolicy readPolicy() const override { return ReadPolicy::Master; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - bool latencyInMicros() const override { return false; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 05e822bd9074..08b0b9224420 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -169,7 +169,6 @@ class RedisHealthCheckerTest EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); EXPECT_FALSE(session->enableCommandStats()); - EXPECT_FALSE(session->latencyInMicros()); session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 54b168f7266fce2cdeebe9a8a6fcf9919471b9f8 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 20 Aug 2019 10:30:49 -0700 Subject: [PATCH 28/55] lowercase commands for metrics Signed-off-by: Nicolas Flacco --- source/extensions/filters/network/common/redis/BUILD | 1 + .../extensions/filters/network/common/redis/client_impl.cc | 7 ++++--- .../extensions/filters/network/common/redis/client_impl.h | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 52ccdf23b139..ce896522c4c2 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -90,6 +90,7 @@ envoy_cc_library( ":supported_commands_lib", "//include/envoy/stats:stats_interface", "//include/envoy/stats:timespan", + "//source/common/common:utility_lib", "//source/common/stats:symbol_table_lib", ], ) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 00438320f1db..58040acf1a45 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -99,12 +99,13 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca const bool empty_buffer = encoder_buffer_.length() == 0; - std::string command = redis_command_stats_->getCommandFromRequest(request); - pending_requests_.emplace_back(*this, callbacks, command); + std::string to_lower_command(redis_command_stats_->getCommandFromRequest(request)); + to_lower_table_.toLowerCase(to_lower_command); + pending_requests_.emplace_back(*this, callbacks, to_lower_command); encoder_->encode(request, encoder_buffer_); if (config_.enableCommandStats()) { - redis_command_stats_->updateStatsTotal(command); + redis_command_stats_->updateStatsTotal(to_lower_command); } // If buffer is full, flush. If the buffer was empty before the request, start the timer. diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 4f1394763500..6e07eff534ec 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -9,6 +9,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hash.h" +#include "common/common/to_lower_table.h" #include "common/network/filter_impl.h" #include "common/protobuf/utility.h" #include "common/singleton/const_singleton.h" @@ -139,6 +140,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr flush_timer_; Envoy::TimeSource& time_source_; const RedisCommandStatsPtr redis_command_stats_; + const ToLowerTable to_lower_table_; }; class ClientFactoryImpl : public ClientFactory { From 0f93e0371195c870628624c7ac978adee0e775cd Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 20 Aug 2019 10:43:44 -0700 Subject: [PATCH 29/55] bit more docs Signed-off-by: Nicolas Flacco --- docs/root/configuration/network_filters/redis_proxy_filter.rst | 2 +- docs/root/intro/version_history.rst | 1 + source/extensions/filters/network/common/redis/client_impl.cc | 2 +- test/extensions/health_checkers/redis/redis_test.cc | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/network_filters/redis_proxy_filter.rst b/docs/root/configuration/network_filters/redis_proxy_filter.rst index 1a94abb013b1..8a403df958df 100644 --- a/docs/root/configuration/network_filters/redis_proxy_filter.rst +++ b/docs/root/configuration/network_filters/redis_proxy_filter.rst @@ -47,7 +47,7 @@ The Redis filter will gather statistics for the command splitter in the Per command statistics ---------------------- -The Redis filter will gather statistics for commands in the *redis..upstream_commands..* namespace. +The Redis filter will gather statistics for commands in the *redis..upstream_commands..* namespace. Additionally, it will collect the aggegate request time to the upstream in *redis..upstream_commands.upstream_rq_time. .. csv-table:: :header: Name, Type, Description diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 83998544fdf1..09d48db98c98 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -28,6 +28,7 @@ Version history * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. +* redis: added :ref:`enable_command_stats ` to enable per command statistics for upstream clusters. * rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 58040acf1a45..64d92f957439 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -201,9 +201,9 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { if (redis_command_stats_->enabled()) { bool success = !canceled && (value->type() != Common::Redis::RespType::Error); redis_command_stats_->updateStats(success, request.command_); - request.aggregate_request_timer_->complete(); request.command_request_timer_->complete(); } + request.aggregate_request_timer_->complete(); PoolCallbacks& callbacks = request.callbacks_; diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 08b0b9224420..ae6c6daa53a2 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -169,7 +169,6 @@ class RedisHealthCheckerTest EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); EXPECT_FALSE(session->enableCommandStats()); - session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 7f0001c817322d0271363c77e6314b094a4ba13c Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 20 Aug 2019 10:57:36 -0700 Subject: [PATCH 30/55] more docs Signed-off-by: Nicolas Flacco --- docs/root/intro/arch_overview/other_protocols/redis.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 9547dc0df245..43b7a917baad 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -90,7 +90,11 @@ Every Redis cluster has its own extra statistics tree rooted at *cluster.. max_upstream_unknown_connections_reached, Counter, Total number of times that an upstream connection to an unknown host is not created after redirection having reached the connection pool's max_upstream_unknown_connections limit upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed - upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests + upstream_commands.upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests + upstream_commands.[command].success, Counter, Total number of successful requests for a specific Redis command + upstream_commands.[command].error, Counter, Total number of failed or cancelled requests for a specific Redis command + upstream_commands.[command].total, Counter, Total number of requests for a specific Redis command (sum of success and error) + upstream_commands.[command].latency, Histogram, Latency of requests for a specific Redis command Supported commands ------------------ From d04ef2991d832f1adfb2b24dabddf43f81f5e432 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 20 Aug 2019 11:34:35 -0700 Subject: [PATCH 31/55] fix docs Signed-off-by: Nicolas Flacco --- .../root/configuration/network_filters/redis_proxy_filter.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/root/configuration/network_filters/redis_proxy_filter.rst b/docs/root/configuration/network_filters/redis_proxy_filter.rst index 8a403df958df..1dd12ee50ad5 100644 --- a/docs/root/configuration/network_filters/redis_proxy_filter.rst +++ b/docs/root/configuration/network_filters/redis_proxy_filter.rst @@ -47,7 +47,9 @@ The Redis filter will gather statistics for the command splitter in the Per command statistics ---------------------- -The Redis filter will gather statistics for commands in the *redis..upstream_commands..* namespace. Additionally, it will collect the aggegate request time to the upstream in *redis..upstream_commands.upstream_rq_time. +The Redis filter will gather statistics for commands in the +*redis..command..* namespace. By default latency stats are in milliseconds and can be +changed to microseconds by setting the configuration parameter :ref:`latency_in_micros ` to true. .. csv-table:: :header: Name, Type, Description From 4a028aec1a15a35bb21e00bcb1b32bf598f7fafe Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 13:27:13 -0700 Subject: [PATCH 32/55] address henry comments Signed-off-by: Nicolas Flacco --- .../network/common/redis/redis_command_stats.cc | 10 +++++----- .../filters/network/common/redis/redis_command_stats.h | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index bddad0e81259..d2b2e490559f 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -81,21 +81,21 @@ std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { case RespType::Array: return getCommandFromRequest(request.asArray().front()); case RespType::Integer: - return std::to_string(request.asInteger()); + return unknown_metric_; case RespType::Null: - return "null"; + return null_metric_; default: return request.asString(); } } -void RedisCommandStats::updateStatsTotal(std::string command) { counter(command + ".total").inc(); } +void RedisCommandStats::updateStatsTotal(std::string command) { counter(command + total_suffix_).inc(); } void RedisCommandStats::updateStats(const bool success, std::string command) { if (success) { - counter(command + ".success").inc(); + counter(command + success_suffix_).inc(); } else { - counter(command + ".error").inc(); + counter(command + error_suffix_).inc(); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 053b0d295c13..25bca868fa7a 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -40,6 +40,11 @@ class RedisCommandStats { const Stats::StatName prefix_; bool enabled_; const std::string latency_suffix_ = ".latency"; + const std::string total_suffix_ = ".total"; + const std::string success_suffix_ = ".success"; + const std::string error_suffix_ = ".error"; + const std::string null_metric_ = "null"; + const std::string unknown_metric_ = "unknown"; public: const Stats::StatName upstream_rq_time_; From b25b4d2775c271184a1042ae36aa4a1049c2ec89 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 16:58:23 -0700 Subject: [PATCH 33/55] simplify slightly Signed-off-by: Nicolas Flacco --- .../common/redis/redis_command_stats.cc | 40 ++----------------- .../common/redis/redis_command_stats.h | 5 ++- 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index d2b2e490559f..cd767033fa13 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,42 +8,8 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) - : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")) { - // Note: Even if this is disabled, we track the upstream_rq_time. - if (enabled_) { - // Create StatName for each Redis command. Note that we don't include Auth or Ping. - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - createStats(command); - } - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - createStats(command); - } - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: - hashMultipleSumResultCommands()) { - createStats(command); - } - createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); - } -} - -void RedisCommandStats::createStats(std::string name) { - stat_name_set_.add(name + ".total"); - stat_name_set_.add(name + ".success"); - stat_name_set_.add(name + ".error"); - stat_name_set_.add(name + ".latency"); -} - Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatName name) { - Stats::StatNameVec names_with_prefix; - names_with_prefix.reserve(2); - names_with_prefix.push_back(prefix_); - names_with_prefix.insert(names_with_prefix.end(), name); - return scope_.symbolTable().join(names_with_prefix); + return scope_.symbolTable().join({prefix_, name}); } Stats::Counter& RedisCommandStats::counter(std::string name) { @@ -89,7 +55,9 @@ std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { } } -void RedisCommandStats::updateStatsTotal(std::string command) { counter(command + total_suffix_).inc(); } +void RedisCommandStats::updateStatsTotal(std::string command) { + counter(command + total_suffix_).inc(); +} void RedisCommandStats::updateStats(const bool success, std::string command) { if (success) { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 25bca868fa7a..bafdcdf2d3a1 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,7 +18,9 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled); + RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) + : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), + enabled_(enabled), upstream_rq_time_(stat_name_set_.add(prefix)) {} Stats::Counter& counter(std::string name); Stats::Histogram& histogram(std::string name); @@ -32,7 +34,6 @@ class RedisCommandStats { bool enabled() { return enabled_; } private: - void createStats(std::string name); Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); Stats::Scope& scope_; From becaba2ba0a0e357983952c0730d530ef0f89605 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 17:13:47 -0700 Subject: [PATCH 34/55] use builtins Signed-off-by: Nicolas Flacco --- .../common/redis/redis_command_stats.cc | 32 +++++++++++++++++++ .../common/redis/redis_command_stats.h | 6 ++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index cd767033fa13..31c437e91497 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,6 +8,38 @@ namespace NetworkFilters { namespace Common { namespace Redis { +RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) + : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), + enabled_(enabled) { + // Note: Even if this is disabled, we track the upstream_rq_time. + stat_name_set_.rememberBuiltin(upstream_rq_time_metric_); + + if (enabled_) { + // Create StatName for each Redis command. Note that we don't include Auth or Ping. + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + createStats(command); + } + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + createStats(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()) { + createStats(command); + } + createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); + } +} + +void RedisCommandStats::createStats(std::string name) { + stat_name_set_.rememberBuiltin(name + ".total"); + stat_name_set_.rememberBuiltin(name + ".success"); + stat_name_set_.rememberBuiltin(name + ".error"); + stat_name_set_.rememberBuiltin(name + ".latency"); +} + Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatName name) { return scope_.symbolTable().join({prefix_, name}); } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index bafdcdf2d3a1..8cec17eb939e 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -18,9 +18,7 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) - : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - enabled_(enabled), upstream_rq_time_(stat_name_set_.add(prefix)) {} + RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled); Stats::Counter& counter(std::string name); Stats::Histogram& histogram(std::string name); @@ -34,6 +32,7 @@ class RedisCommandStats { bool enabled() { return enabled_; } private: + void createStats(std::string name); Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); Stats::Scope& scope_; @@ -46,6 +45,7 @@ class RedisCommandStats { const std::string error_suffix_ = ".error"; const std::string null_metric_ = "null"; const std::string unknown_metric_ = "unknown"; + const std::string upstream_rq_time_metric_ = "upstream_rq_time"; public: const Stats::StatName upstream_rq_time_; From f6ec8bf14380d693f79bb27704c5c3d775e7baa1 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 17:28:22 -0700 Subject: [PATCH 35/55] updated Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/redis_command_stats.cc | 2 +- .../filters/network/common/redis/redis_command_stats.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 31c437e91497..796a063b305b 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -10,7 +10,7 @@ namespace Redis { RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - enabled_(enabled) { + enabled_(enabled), upstream_rq_time_(stat_name_set_.getStatName(upstream_rq_time_metric_)) { // Note: Even if this is disabled, we track the upstream_rq_time. stat_name_set_.rememberBuiltin(upstream_rq_time_metric_); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 8cec17eb939e..b9ba81723349 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -46,8 +46,6 @@ class RedisCommandStats { const std::string null_metric_ = "null"; const std::string unknown_metric_ = "unknown"; const std::string upstream_rq_time_metric_ = "upstream_rq_time"; - -public: const Stats::StatName upstream_rq_time_; }; using RedisCommandStatsPtr = std::shared_ptr; From 703d0feb132022521be10162d976c0a268795804 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 17:35:43 -0700 Subject: [PATCH 36/55] Kick CI Signed-off-by: Nicolas Flacco From fe218a7cfa7ed3b0a9162718f839075200b5918b Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 21 Aug 2019 17:36:10 -0700 Subject: [PATCH 37/55] Kick CI Signed-off-by: Nicolas Flacco From 7f703928590a368c2c13e419c9d0c31b9dc807de Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 23 Aug 2019 10:54:42 -0700 Subject: [PATCH 38/55] Per jmarantz comments Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/client_impl.cc | 8 ++++---- .../extensions/filters/network/common/redis/client_impl.h | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 64d92f957439..b8bddbd9f002 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -52,9 +52,9 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche const Config& config) { auto redis_command_stats = std::make_shared( host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); - std::unique_ptr client(new ClientImpl(host, dispatcher, std::move(encoder), + auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, config, - std::move(redis_command_stats))); + std::move(redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -68,8 +68,8 @@ ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dis RedisCommandStatsPtr&& redis_command_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), - connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), - flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })), + connect_or_op_timer_(dispatcher.createTimer([this]() { onConnectOrOpTimeout(); })), + flush_timer_(dispatcher.createTimer([this]() { flushBufferAndResetTimer(); })), time_source_(dispatcher.timeSource()), redis_command_stats_(redis_command_stats) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 6e07eff534ec..081d7d507b1b 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -72,6 +72,9 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config); + ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, + DecoderFactory& decoder_factory, const Config& config, + RedisCommandStatsPtr&& redis_command_stats); ~ClientImpl() override; // Client @@ -113,9 +116,6 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Stats::CompletableTimespanPtr command_request_timer_; }; - ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, - DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsPtr&& redis_command_stats); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); From 66cb5fd95f35324dcade21a28f7035b31b775023 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Mon, 26 Aug 2019 16:45:12 -0700 Subject: [PATCH 39/55] respond to comments Signed-off-by: Nicolas Flacco --- .../arch_overview/other_protocols/redis.rst | 7 ++ .../network/common/redis/client_impl.cc | 5 +- .../common/redis/redis_command_stats.cc | 71 ++++++++----------- .../common/redis/redis_command_stats.h | 25 +++---- 4 files changed, 49 insertions(+), 59 deletions(-) diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 43b7a917baad..980cc603217e 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -91,6 +91,13 @@ Every Redis cluster has its own extra statistics tree rooted at *cluster.. max_upstream_unknown_connections_reached, Counter, Total number of times that an upstream connection to an unknown host is not created after redirection having reached the connection pool's max_upstream_unknown_connections limit upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed upstream_commands.upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests + +Per-cluster command statistics can be enabled via the setting :ref:`enable_command_stats `: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + upstream_commands.[command].success, Counter, Total number of successful requests for a specific Redis command upstream_commands.[command].error, Counter, Total number of failed or cancelled requests for a specific Redis command upstream_commands.[command].total, Counter, Total number of requests for a specific Redis command (sum of success and error) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index b8bddbd9f002..b2f8964cd5b6 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -52,9 +52,8 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche const Config& config) { auto redis_command_stats = std::make_shared( host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); - auto client = std::make_unique(host, dispatcher, std::move(encoder), - decoder_factory, config, - std::move(redis_command_stats)); + auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, + config, std::move(redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 796a063b305b..db5eb23ef065 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -10,67 +10,54 @@ namespace Redis { RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - enabled_(enabled), upstream_rq_time_(stat_name_set_.getStatName(upstream_rq_time_metric_)) { + enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), + latency_(stat_name_set_.add("latency")), total_(stat_name_set_.add("total")), + success_(stat_name_set_.add("success")), error_(stat_name_set_.add("error")) { // Note: Even if this is disabled, we track the upstream_rq_time. - stat_name_set_.rememberBuiltin(upstream_rq_time_metric_); - if (enabled_) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - createStats(command); + stat_name_set_.rememberBuiltin(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - createStats(command); + stat_name_set_.rememberBuiltin(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: hashMultipleSumResultCommands()) { - createStats(command); + stat_name_set_.rememberBuiltin(command); } - createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - createStats(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); + stat_name_set_.rememberBuiltin( + Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + stat_name_set_.rememberBuiltin( + Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } } -void RedisCommandStats::createStats(std::string name) { - stat_name_set_.rememberBuiltin(name + ".total"); - stat_name_set_.rememberBuiltin(name + ".success"); - stat_name_set_.rememberBuiltin(name + ".error"); - stat_name_set_.rememberBuiltin(name + ".latency"); -} - -Stats::SymbolTable::StoragePtr RedisCommandStats::addPrefix(const Stats::StatName name) { - return scope_.symbolTable().join({prefix_, name}); -} - -Stats::Counter& RedisCommandStats::counter(std::string name) { - Stats::StatName stat_name = stat_name_set_.getStatName(name); - const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(stat_name); - return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); -} - -Stats::Histogram& RedisCommandStats::histogram(std::string name) { - Stats::StatName stat_name = stat_name_set_.getStatName(name); - return histogram(stat_name); +Stats::Counter& RedisCommandStats::counter(const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = scope_.symbolTable().join(stat_names); + Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); + return scope_.counterFromStatName(full_stat_name); } -Stats::Histogram& RedisCommandStats::histogram(Stats::StatName stat_name) { - const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(stat_name); - return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); +Stats::Histogram& RedisCommandStats::histogram(const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = scope_.symbolTable().join(stat_names); + Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); + return scope_.histogramFromStatName(full_stat_name); } Stats::CompletableTimespanPtr -RedisCommandStats::createCommandTimer(std::string name, Envoy::TimeSource& time_source) { - Stats::StatName stat_name = stat_name_set_.getStatName(name + latency_suffix_); - return std::make_unique>(histogram(stat_name), - time_source); +RedisCommandStats::createCommandTimer(std::string command, Envoy::TimeSource& time_source) { + Stats::StatName stat_name = stat_name_set_.getStatName(command); + return std::make_unique>( + histogram({prefix_, stat_name, latency_}), time_source); } Stats::CompletableTimespanPtr RedisCommandStats::createAggregateTimer(Envoy::TimeSource& time_source) { return std::make_unique>( - histogram(upstream_rq_time_), time_source); + histogram({prefix_, upstream_rq_time_}), time_source); } std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { @@ -87,15 +74,17 @@ std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { } } -void RedisCommandStats::updateStatsTotal(std::string command) { - counter(command + total_suffix_).inc(); +void RedisCommandStats::updateStatsTotal(const std::string& command) { + Stats::StatName stat_name = stat_name_set_.getStatName(command); + counter({prefix_, stat_name, total_}).inc(); } -void RedisCommandStats::updateStats(const bool success, std::string command) { +void RedisCommandStats::updateStats(const bool success, const std::string& command) { + Stats::StatName stat_name = stat_name_set_.getStatName(command); if (success) { - counter(command + success_suffix_).inc(); + counter({prefix_, stat_name, success_}).inc(); } else { - counter(command + error_suffix_).inc(); + counter({prefix_, stat_name, success_}).inc(); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index b9ba81723349..306586895eb1 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -20,33 +20,28 @@ class RedisCommandStats { public: RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled); - Stats::Counter& counter(std::string name); - Stats::Histogram& histogram(std::string name); - Stats::Histogram& histogram(Stats::StatName stat_name); - Stats::CompletableTimespanPtr createCommandTimer(std::string name, + Stats::Counter& counter(const Stats::StatNameVec& stat_names); + Stats::Histogram& histogram(const Stats::StatNameVec& stat_names); + Stats::CompletableTimespanPtr createCommandTimer(std::string command, Envoy::TimeSource& time_source); Stats::CompletableTimespanPtr createAggregateTimer(Envoy::TimeSource& time_source); std::string getCommandFromRequest(const RespValue& request); - void updateStatsTotal(std::string command); - void updateStats(const bool success, std::string command); + void updateStatsTotal(const std::string& command); + void updateStats(const bool success, const std::string& command); bool enabled() { return enabled_; } private: - void createStats(std::string name); - Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatName name); - Stats::Scope& scope_; Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; bool enabled_; - const std::string latency_suffix_ = ".latency"; - const std::string total_suffix_ = ".total"; - const std::string success_suffix_ = ".success"; - const std::string error_suffix_ = ".error"; + const Stats::StatName upstream_rq_time_; + const Stats::StatName latency_; + const Stats::StatName total_; + const Stats::StatName success_; + const Stats::StatName error_; const std::string null_metric_ = "null"; const std::string unknown_metric_ = "unknown"; - const std::string upstream_rq_time_metric_ = "upstream_rq_time"; - const Stats::StatName upstream_rq_time_; }; using RedisCommandStatsPtr = std::shared_ptr; From 6d7c4b5a2adff446f9d735188d842ddb35352008 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 4 Sep 2019 18:09:52 -0700 Subject: [PATCH 40/55] Update for jmarantz new comments, part 1, basics Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/BUILD | 1 + .../network/common/redis/client_impl.cc | 24 +++++++++++-------- .../network/common/redis/client_impl.h | 6 ++--- .../common/redis/redis_command_stats.cc | 19 ++++++++------- .../common/redis/redis_command_stats.h | 16 ++++++++----- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index ce896522c4c2..8cc5a3991ae8 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -90,6 +90,7 @@ envoy_cc_library( ":supported_commands_lib", "//include/envoy/stats:stats_interface", "//include/envoy/stats:timespan", + "//source/common/common:to_lower_table_lib", "//source/common/common:utility_lib", "//source/common/stats:symbol_table_lib", ], diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 2578b728e499..341c7dc4521b 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -98,15 +98,19 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca const bool empty_buffer = encoder_buffer_.length() == 0; - std::string to_lower_command(redis_command_stats_->getCommandFromRequest(request)); - to_lower_table_.toLowerCase(to_lower_command); - pending_requests_.emplace_back(*this, callbacks, to_lower_command); - encoder_->encode(request, encoder_buffer_); - + Stats::StatName stat_name; if (config_.enableCommandStats()) { - redis_command_stats_->updateStatsTotal(to_lower_command); + // Only lowercase command and get stat_name if we enable command stats + stat_name = redis_command_stats_->getCommandFromRequest(request); + redis_command_stats_->updateStatsTotal(stat_name); + } else { + // If disabled, we use a placeholder stat name "unused" that is not used + stat_name = redis_command_stats_->getUnusedStatName(); } + pending_requests_.emplace_back(*this, callbacks, stat_name); + encoder_->encode(request, encoder_buffer_); + // If buffer is full, flush. If the buffer was empty before the request, start the timer. if (encoder_buffer_.length() >= config_.maxBufferSizeBeforeFlush()) { flushBufferAndResetTimer(); @@ -199,7 +203,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { if (redis_command_stats_->enabled()) { bool success = !canceled && (value->type() != Common::Redis::RespType::Error); - redis_command_stats_->updateStats(success, request.command_); + redis_command_stats_->updateStats(success, request.stat_name_); request.command_request_timer_->complete(); } request.aggregate_request_timer_->complete(); @@ -244,12 +248,12 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, - std::string command) - : parent_(parent), callbacks_(callbacks), command_{command}, + Stats::StatName stat_name) + : parent_(parent), callbacks_(callbacks), stat_name_{stat_name}, aggregate_request_timer_( parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)), command_request_timer_( - parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_)) { + parent_.redis_command_stats_->createCommandTimer(stat_name_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 081d7d507b1b..194d30d37293 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -9,7 +9,6 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hash.h" -#include "common/common/to_lower_table.h" #include "common/network/filter_impl.h" #include "common/protobuf/utility.h" #include "common/singleton/const_singleton.h" @@ -102,7 +101,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne }; struct PendingRequest : public PoolRequest { - PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, std::string command); + PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, Stats::StatName stat_name); ~PendingRequest() override; // PoolRequest @@ -110,7 +109,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl& parent_; PoolCallbacks& callbacks_; - std::string command_; + Stats::StatName stat_name_; bool canceled_{}; Stats::CompletableTimespanPtr aggregate_request_timer_; Stats::CompletableTimespanPtr command_request_timer_; @@ -140,7 +139,6 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr flush_timer_; Envoy::TimeSource& time_source_; const RedisCommandStatsPtr redis_command_stats_; - const ToLowerTable to_lower_table_; }; class ClientFactoryImpl : public ClientFactory { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index db5eb23ef065..ceadd2d9d177 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -12,7 +12,9 @@ RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& pre : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), latency_(stat_name_set_.add("latency")), total_(stat_name_set_.add("total")), - success_(stat_name_set_.add("success")), error_(stat_name_set_.add("error")) { + success_(stat_name_set_.add("success")), error_(stat_name_set_.add("error")), + unused_metric_(stat_name_set_.add("unused")), null_metric_(stat_name_set_.add("null")), + unknown_metric_(stat_name_set_.add("unknown")) { // Note: Even if this is disabled, we track the upstream_rq_time. if (enabled_) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. @@ -48,8 +50,7 @@ Stats::Histogram& RedisCommandStats::histogram(const Stats::StatNameVec& stat_na } Stats::CompletableTimespanPtr -RedisCommandStats::createCommandTimer(std::string command, Envoy::TimeSource& time_source) { - Stats::StatName stat_name = stat_name_set_.getStatName(command); +RedisCommandStats::createCommandTimer(Stats::StatName stat_name, Envoy::TimeSource& time_source) { return std::make_unique>( histogram({prefix_, stat_name, latency_}), time_source); } @@ -60,7 +61,7 @@ RedisCommandStats::createAggregateTimer(Envoy::TimeSource& time_source) { histogram({prefix_, upstream_rq_time_}), time_source); } -std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { +Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& request) { // Get command from RespValue switch (request.type()) { case RespType::Array: @@ -70,17 +71,17 @@ std::string RedisCommandStats::getCommandFromRequest(const RespValue& request) { case RespType::Null: return null_metric_; default: - return request.asString(); + std::string to_lower_command(request.asString()); + to_lower_table_.toLowerCase(to_lower_command); + return stat_name_set_.getStatName(to_lower_command); } } -void RedisCommandStats::updateStatsTotal(const std::string& command) { - Stats::StatName stat_name = stat_name_set_.getStatName(command); +void RedisCommandStats::updateStatsTotal(Stats::StatName stat_name) { counter({prefix_, stat_name, total_}).inc(); } -void RedisCommandStats::updateStats(const bool success, const std::string& command) { - Stats::StatName stat_name = stat_name_set_.getStatName(command); +void RedisCommandStats::updateStats(const bool success, Stats::StatName stat_name) { if (success) { counter({prefix_, stat_name, success_}).inc(); } else { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 306586895eb1..ffb227d4ce73 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -7,6 +7,7 @@ #include "envoy/stats/timespan.h" #include "common/stats/symbol_table_impl.h" +#include "common/common/to_lower_table.h" #include "extensions/filters/network/common/redis/codec.h" @@ -22,13 +23,14 @@ class RedisCommandStats { Stats::Counter& counter(const Stats::StatNameVec& stat_names); Stats::Histogram& histogram(const Stats::StatNameVec& stat_names); - Stats::CompletableTimespanPtr createCommandTimer(std::string command, + Stats::CompletableTimespanPtr createCommandTimer(Stats::StatName stat_name, Envoy::TimeSource& time_source); Stats::CompletableTimespanPtr createAggregateTimer(Envoy::TimeSource& time_source); - std::string getCommandFromRequest(const RespValue& request); - void updateStatsTotal(const std::string& command); - void updateStats(const bool success, const std::string& command); + Stats::StatName getCommandFromRequest(const RespValue& request); + void updateStatsTotal(Stats::StatName stat_name); + void updateStats(const bool success, Stats::StatName stat_name); bool enabled() { return enabled_; } + Stats::StatName getUnusedStatName() {return unused_metric_; } private: Stats::Scope& scope_; @@ -40,8 +42,10 @@ class RedisCommandStats { const Stats::StatName total_; const Stats::StatName success_; const Stats::StatName error_; - const std::string null_metric_ = "null"; - const std::string unknown_metric_ = "unknown"; + const Stats::StatName unused_metric_; + const Stats::StatName null_metric_; + const Stats::StatName unknown_metric_; + const ToLowerTable to_lower_table_; }; using RedisCommandStatsPtr = std::shared_ptr; From 8d9d3d134bc04e85f553b573a3578f4a3ab71722 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 4 Sep 2019 18:37:13 -0700 Subject: [PATCH 41/55] Update for jmarantz new comments, part 2, use stat_name_pool and stat_name_map Signed-off-by: Nicolas Flacco --- .../common/redis/redis_command_stats.cc | 38 +++++++++++++------ .../common/redis/redis_command_stats.h | 5 ++- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index ceadd2d9d177..38c49e0edfaf 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -9,34 +9,40 @@ namespace Common { namespace Redis { RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) - : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - enabled_(enabled), upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), - latency_(stat_name_set_.add("latency")), total_(stat_name_set_.add("total")), - success_(stat_name_set_.add("success")), error_(stat_name_set_.add("error")), - unused_metric_(stat_name_set_.add("unused")), null_metric_(stat_name_set_.add("null")), - unknown_metric_(stat_name_set_.add("unknown")) { + : scope_(scope), stat_name_pool_(scope.symbolTable()), + prefix_(stat_name_pool_.add(prefix)), + enabled_(enabled), upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), + latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), + success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), + unused_metric_(stat_name_pool_.add("unused")), null_metric_(stat_name_pool_.add("null")), + unknown_metric_(stat_name_pool_.add("unknown")) { // Note: Even if this is disabled, we track the upstream_rq_time. if (enabled_) { // Create StatName for each Redis command. Note that we don't include Auth or Ping. for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - stat_name_set_.rememberBuiltin(command); + addCommandToPool(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - stat_name_set_.rememberBuiltin(command); + addCommandToPool(command); } for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: hashMultipleSumResultCommands()) { - stat_name_set_.rememberBuiltin(command); + addCommandToPool(command); } - stat_name_set_.rememberBuiltin( + addCommandToPool( Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - stat_name_set_.rememberBuiltin( + addCommandToPool( Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } } +void RedisCommandStats::addCommandToPool(const std::string& command) { + Stats::StatName stat_name = stat_name_pool_.add(command); + stat_name_map_[std::string(command)] = stat_name; +} + Stats::Counter& RedisCommandStats::counter(const Stats::StatNameVec& stat_names) { const Stats::SymbolTable::StoragePtr storage_ptr = scope_.symbolTable().join(stat_names); Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); @@ -71,9 +77,17 @@ Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& reques case RespType::Null: return null_metric_; default: + // Once we have a RespType::String we lowercase it and then look it up in our stat_name_map. + // If it does not exist, we return our unknown stat name. std::string to_lower_command(request.asString()); to_lower_table_.toLowerCase(to_lower_command); - return stat_name_set_.getStatName(to_lower_command); + + auto iter = stat_name_map_.find(to_lower_command); + if (iter != stat_name_map_.end()) { + return iter->second; + } else { + return unknown_metric_; + } } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index ffb227d4ce73..1b0894ebd60b 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -33,8 +33,11 @@ class RedisCommandStats { Stats::StatName getUnusedStatName() {return unused_metric_; } private: + void addCommandToPool(const std::string& command); + Stats::Scope& scope_; - Stats::StatNameSet stat_name_set_; + Stats::StatNamePool stat_name_pool_; + StringMap stat_name_map_; const Stats::StatName prefix_; bool enabled_; const Stats::StatName upstream_rq_time_; From bc5a96f83fd3c6d5d90844858786d21b1fb40205 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Wed, 4 Sep 2019 18:39:48 -0700 Subject: [PATCH 42/55] fix formatting Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/redis_command_stats.cc | 9 +++------ .../filters/network/common/redis/redis_command_stats.h | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 38c49e0edfaf..e51c8a4d5483 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -9,8 +9,7 @@ namespace Common { namespace Redis { RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) - : scope_(scope), stat_name_pool_(scope.symbolTable()), - prefix_(stat_name_pool_.add(prefix)), + : scope_(scope), stat_name_pool_(scope.symbolTable()), prefix_(stat_name_pool_.add(prefix)), enabled_(enabled), upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), @@ -31,10 +30,8 @@ RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& pre hashMultipleSumResultCommands()) { addCommandToPool(command); } - addCommandToPool( - Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - addCommandToPool( - Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 1b0894ebd60b..1386f8e3b997 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -6,8 +6,8 @@ #include "envoy/stats/scope.h" #include "envoy/stats/timespan.h" -#include "common/stats/symbol_table_impl.h" #include "common/common/to_lower_table.h" +#include "common/stats/symbol_table_impl.h" #include "extensions/filters/network/common/redis/codec.h" @@ -30,7 +30,7 @@ class RedisCommandStats { void updateStatsTotal(Stats::StatName stat_name); void updateStats(const bool success, Stats::StatName stat_name); bool enabled() { return enabled_; } - Stats::StatName getUnusedStatName() {return unused_metric_; } + Stats::StatName getUnusedStatName() { return unused_metric_; } private: void addCommandToPool(const std::string& command); From 4e82245caa4c0ad347e63e74157b32ca0ccdddbd Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 5 Sep 2019 10:20:35 -0700 Subject: [PATCH 43/55] update stat_name to command Signed-off-by: Nicolas Flacco --- .../network/common/redis/client_impl.cc | 20 +++++++++---------- .../network/common/redis/client_impl.h | 2 +- .../common/redis/redis_command_stats.cc | 20 +++++++++---------- .../common/redis/redis_command_stats.h | 8 ++++---- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 341c7dc4521b..83cadc8e6391 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -98,17 +98,17 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca const bool empty_buffer = encoder_buffer_.length() == 0; - Stats::StatName stat_name; + Stats::StatName command; if (config_.enableCommandStats()) { - // Only lowercase command and get stat_name if we enable command stats - stat_name = redis_command_stats_->getCommandFromRequest(request); - redis_command_stats_->updateStatsTotal(stat_name); + // Only lowercase command and get StatName if we enable command stats + command = redis_command_stats_->getCommandFromRequest(request); + redis_command_stats_->updateStatsTotal(command); } else { // If disabled, we use a placeholder stat name "unused" that is not used - stat_name = redis_command_stats_->getUnusedStatName(); + command = redis_command_stats_->getUnusedStatName(); } - pending_requests_.emplace_back(*this, callbacks, stat_name); + pending_requests_.emplace_back(*this, callbacks, command); encoder_->encode(request, encoder_buffer_); // If buffer is full, flush. If the buffer was empty before the request, start the timer. @@ -203,7 +203,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { if (redis_command_stats_->enabled()) { bool success = !canceled && (value->type() != Common::Redis::RespType::Error); - redis_command_stats_->updateStats(success, request.stat_name_); + redis_command_stats_->updateStats(success, request.command_); request.command_request_timer_->complete(); } request.aggregate_request_timer_->complete(); @@ -248,12 +248,12 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, - Stats::StatName stat_name) - : parent_(parent), callbacks_(callbacks), stat_name_{stat_name}, + Stats::StatName command) + : parent_(parent), callbacks_(callbacks), command_{command}, aggregate_request_timer_( parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)), command_request_timer_( - parent_.redis_command_stats_->createCommandTimer(stat_name_, parent_.time_source_)) { + parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_)) { parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 194d30d37293..1ddf03ffad81 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -109,7 +109,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl& parent_; PoolCallbacks& callbacks_; - Stats::StatName stat_name_; + Stats::StatName command_; bool canceled_{}; Stats::CompletableTimespanPtr aggregate_request_timer_; Stats::CompletableTimespanPtr command_request_timer_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index e51c8a4d5483..1a80f0d9869e 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -35,9 +35,9 @@ RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& pre } } -void RedisCommandStats::addCommandToPool(const std::string& command) { - Stats::StatName stat_name = stat_name_pool_.add(command); - stat_name_map_[std::string(command)] = stat_name; +void RedisCommandStats::addCommandToPool(const std::string& command_string) { + Stats::StatName command = stat_name_pool_.add(command_string); + stat_name_map_[command_string] = command; } Stats::Counter& RedisCommandStats::counter(const Stats::StatNameVec& stat_names) { @@ -53,9 +53,9 @@ Stats::Histogram& RedisCommandStats::histogram(const Stats::StatNameVec& stat_na } Stats::CompletableTimespanPtr -RedisCommandStats::createCommandTimer(Stats::StatName stat_name, Envoy::TimeSource& time_source) { +RedisCommandStats::createCommandTimer(Stats::StatName command, Envoy::TimeSource& time_source) { return std::make_unique>( - histogram({prefix_, stat_name, latency_}), time_source); + histogram({prefix_, command, latency_}), time_source); } Stats::CompletableTimespanPtr @@ -88,15 +88,15 @@ Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& reques } } -void RedisCommandStats::updateStatsTotal(Stats::StatName stat_name) { - counter({prefix_, stat_name, total_}).inc(); +void RedisCommandStats::updateStatsTotal(Stats::StatName command) { + counter({prefix_, command, total_}).inc(); } -void RedisCommandStats::updateStats(const bool success, Stats::StatName stat_name) { +void RedisCommandStats::updateStats(const bool success, Stats::StatName command) { if (success) { - counter({prefix_, stat_name, success_}).inc(); + counter({prefix_, command, success_}).inc(); } else { - counter({prefix_, stat_name, success_}).inc(); + counter({prefix_, command, success_}).inc(); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 1386f8e3b997..fcfcd0ecb120 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -23,17 +23,17 @@ class RedisCommandStats { Stats::Counter& counter(const Stats::StatNameVec& stat_names); Stats::Histogram& histogram(const Stats::StatNameVec& stat_names); - Stats::CompletableTimespanPtr createCommandTimer(Stats::StatName stat_name, + Stats::CompletableTimespanPtr createCommandTimer(Stats::StatName command, Envoy::TimeSource& time_source); Stats::CompletableTimespanPtr createAggregateTimer(Envoy::TimeSource& time_source); Stats::StatName getCommandFromRequest(const RespValue& request); - void updateStatsTotal(Stats::StatName stat_name); - void updateStats(const bool success, Stats::StatName stat_name); + void updateStatsTotal(Stats::StatName command); + void updateStats(const bool success, Stats::StatName command); bool enabled() { return enabled_; } Stats::StatName getUnusedStatName() { return unused_metric_; } private: - void addCommandToPool(const std::string& command); + void addCommandToPool(const std::string& command_string); Stats::Scope& scope_; Stats::StatNamePool stat_name_pool_; From abf2297e75ba10102011f189c6aa07d152dfa22b Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 5 Sep 2019 16:06:16 -0700 Subject: [PATCH 44/55] attempt #1, move stats to config Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.cc | 7 +++++-- source/extensions/clusters/redis/redis_cluster.h | 1 + .../extensions/filters/network/common/redis/BUILD | 2 +- .../filters/network/common/redis/client.h | 3 ++- .../filters/network/common/redis/client_impl.cc | 9 ++++----- .../filters/network/common/redis/client_impl.h | 5 ++--- .../network/common/redis/redis_command_stats.h | 11 ++++++++++- .../extensions/filters/network/redis_proxy/BUILD | 1 + .../filters/network/redis_proxy/config.cc | 5 ++++- .../filters/network/redis_proxy/conn_pool_impl.cc | 12 ++++++++---- .../filters/network/redis_proxy/conn_pool_impl.h | 4 +++- source/extensions/health_checkers/redis/redis.cc | 10 ++++++++-- source/extensions/health_checkers/redis/redis.h | 1 + .../network/common/redis/client_impl_test.cc | 15 +++++++++++++-- .../network/redis_proxy/conn_pool_impl_test.cc | 7 +++++-- .../health_checkers/redis/redis_test.cc | 3 ++- 16 files changed, 70 insertions(+), 26 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index dc41651b0ca1..f3aabc86c278 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -168,7 +168,9 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : parent_(parent), dispatcher_(parent.dispatcher_), resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), - client_factory_(client_factory), buffer_timeout_(0) {} + client_factory_(client_factory), buffer_timeout_(0), + redis_command_stats_(std::make_shared( + parent_.info()->statsScope(), "upstream_commands", false)) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. @@ -249,7 +251,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = client_factory_.create(host, dispatcher_, *this); + client->client_ = + client_factory_.create(host, dispatcher_, *this, std::move(redis_command_stats_)); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 4fde9bdc5811..094ce5260837 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -242,6 +242,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { Event::TimerPtr resolve_timer_; NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; const std::chrono::milliseconds buffer_timeout_; + NetworkFilters::Common::Redis::RedisCommandStatsPtr redis_command_stats_; }; Upstream::ClusterManager& cluster_manager_; diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 8cc5a3991ae8..336c9f2b303a 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -46,6 +46,7 @@ envoy_cc_library( hdrs = ["client.h"], deps = [ ":codec_lib", + ":redis_command_stats_lib", "//include/envoy/upstream:cluster_manager_interface", ], ) @@ -57,7 +58,6 @@ envoy_cc_library( deps = [ ":client_interface", ":codec_lib", - ":redis_command_stats_lib", "//include/envoy/router:router_interface", "//include/envoy/stats:timespan", "//include/envoy/thread_local:thread_local_interface", diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 1432eb43af8d..21d3e20b635b 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -5,6 +5,7 @@ #include "envoy/upstream/cluster_manager.h" #include "extensions/filters/network/common/redis/codec_impl.h" +#include "extensions/filters/network/common/redis/redis_command_stats.h" namespace Envoy { namespace Extensions { @@ -189,7 +190,7 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) PURE; + const Config& config, RedisCommandStatsPtr&& redis_command_stats) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 83cadc8e6391..5b8847719e97 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -49,9 +49,7 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config) { - auto redis_command_stats = std::make_shared( - host->cluster().statsScope(), "upstream_commands", config.enableCommandStats()); + const Config& config, RedisCommandStatsPtr&& redis_command_stats) { auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, config, std::move(redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; @@ -275,9 +273,10 @@ void ClientImpl::PendingRequest::cancel() { ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, - Event::Dispatcher& dispatcher, const Config& config) { + Event::Dispatcher& dispatcher, const Config& config, + RedisCommandStatsPtr&& redis_command_stats) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config); + config, std::move(redis_command_stats)); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 1ddf03ffad81..72a5fe933a7e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -16,7 +16,6 @@ #include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/client.h" -#include "extensions/filters/network/common/redis/redis_command_stats.h" namespace Envoy { namespace Extensions { @@ -69,7 +68,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config); + const Config& config, RedisCommandStatsPtr&& redis_command_stats); ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, @@ -145,7 +144,7 @@ class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) override; + const Config& config, RedisCommandStatsPtr&& redis_command_stats) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index fcfcd0ecb120..10b40d01315a 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -19,7 +19,16 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled); + RedisCommandStats(Stats::Scope&, const std::string& prefix, bool enabled); + + // TODO: Make std::shared_ptr be RedisCommandStatsPtr + static std::shared_ptr + createRedisCommandStats(Stats::Scope& stats_scope, const std::string& prefix, bool enabled) { + auto redis_command_stats = std::make_shared( + stats_scope, prefix, + enabled); // TODO: createScope uses make_unique, should I be doing that? + return redis_command_stats; + } Stats::Counter& counter(const Stats::StatNameVec& stat_names); Stats::Histogram& histogram(const Stats::StatNameVec& stat_names); diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index bbc18dff95d2..2aee3f55e361 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -120,6 +120,7 @@ envoy_cc_library( "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", "//source/extensions/filters/network/common/redis:codec_lib", + "//source/extensions/filters/network/common/redis:redis_command_stats_lib", "//source/extensions/filters/network/redis_proxy:command_splitter_lib", "//source/extensions/filters/network/redis_proxy:conn_pool_lib", "//source/extensions/filters/network/redis_proxy:proxy_filter_lib", diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 5e3e4018260d..28e85c3f282d 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -61,11 +61,14 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP for (auto& cluster : unique_clusters) { Stats::ScopePtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); + + auto redis_command_stats = std::make_shared( + *stats_scope, "upstream_commands", proto_config.settings().enable_command_stats()); upstreams.emplace(cluster, std::make_shared( cluster, context.clusterManager(), Common::Redis::Client::ClientFactoryImpl::instance_, context.threadLocal(), proto_config.settings(), context.api(), - std::move(stats_scope))); + std::move(stats_scope), std::move(redis_command_stats))); } auto router = diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index a097924ea734..7b317332808f 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -23,10 +23,12 @@ InstanceImpl::InstanceImpl( const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::ScopePtr&& stats_scope) + Api::Api& api, Stats::ScopePtr&& stats_scope, + Common::Redis::RedisCommandStatsPtr&& redis_command_stats) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), - api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(*stats_scope_))} { + api_(api), stats_scope_(std::move(stats_scope)), + redis_command_stats_(std::move(redis_command_stats)), + redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -195,7 +197,9 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_); + client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, + std::move(parent_.redis_command_stats_)); + // client->redis_client_->addRedisCommandStats(parent_.redis_command_stats_); client->redis_client_->addConnectionCallbacks(*client); // TODO(hyang): should the auth command and readonly command be moved to the factory method? if (!auth_password_.empty()) { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 91ea51f3e752..b4a31155b9d7 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -51,7 +51,8 @@ class InstanceImpl : public Instance { const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::ScopePtr&& stats_scope); + Api::Api& api, Stats::ScopePtr&& stats_scope, + Common::Redis::RedisCommandStatsPtr&& redis_command_stats); // RedisProxy::ConnPool::Instance Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, @@ -131,6 +132,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ConfigImpl config_; Api::Api& api_; Stats::ScopePtr stats_scope_; + Common::Redis::RedisCommandStatsPtr redis_command_stats_; RedisClusterStats redis_cluster_stats_; }; diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 9f130824639c..acc6204e8fa5 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -22,7 +22,12 @@ RedisHealthChecker::RedisHealthChecker( RedisHealthChecker::RedisActiveHealthCheckSession::RedisActiveHealthCheckSession( RedisHealthChecker& parent, const Upstream::HostSharedPtr& host) - : ActiveHealthCheckSession(parent, host), parent_(parent) {} + : ActiveHealthCheckSession(parent, host), parent_(parent) { + auto redis_command_stats = + std::make_shared( + parent_.cluster_.info()->statsScope(), "upstream_commands", false); + redis_command_stats_ = std::move(redis_command_stats); +} RedisHealthChecker::RedisActiveHealthCheckSession::~RedisActiveHealthCheckSession() { ASSERT(current_request_ == nullptr); @@ -51,7 +56,8 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this); + client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, + std::move(redis_command_stats_)); client_->addConnectionCallbacks(*this); } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index d5d33542003f..61106195bc19 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -96,6 +96,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { RedisHealthChecker& parent_; Extensions::NetworkFilters::Common::Redis::Client::ClientPtr client_; Extensions::NetworkFilters::Common::Redis::Client::PoolRequest* current_request_{}; + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr redis_command_stats_; }; enum class Type { Ping, Exists }; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 03cc64f365ee..9170c9cb7798 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -73,8 +73,12 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, connect()); EXPECT_CALL(*upstream_connection_, noDelay(true)); + stats_scope_ = stats_.createScope("test"); + redis_command_stats_ = Common::Redis::RedisCommandStats::createRedisCommandStats( + *stats_scope_, "upstream_commands", true); + client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_); + *config_, std::move(redis_command_stats_)); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -111,6 +115,9 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Network::ReadFilterSharedPtr upstream_read_filter_; std::unique_ptr config_; ClientPtr client_; + Stats::IsolatedStoreImpl stats_; + Stats::ScopePtr stats_scope_; + Common::Redis::RedisCommandStatsPtr redis_command_stats_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { @@ -877,7 +884,11 @@ TEST(RedisClientFactoryImplTest, Basic) { EXPECT_CALL(*host, createConnection_(_, _)).WillOnce(Return(conn_info)); NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); - ClientPtr client = factory.create(host, dispatcher, config); + Stats::IsolatedStoreImpl stats_; + Stats::ScopePtr stats_scope = stats_.createScope("test"); + auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( + *stats_scope, "upstream_commands", true); + ClientPtr client = factory.create(host, dispatcher, config, std::move(redis_command_stats)); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 9dd249ec6002..bdf04bad137a 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -66,11 +66,13 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); + auto redis_command_stats = + std::make_shared(*store, "upstream_commands", true); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( 20, hashtagging, true, max_unknown_conns, read_policy_), - api_, std::move(store)); + api_, std::move(store), std::move(redis_command_stats)); // Set the authentication password for this connection pool. conn_pool_impl->tls_->getTyped().auth_password_ = auth_password_; conn_pool_ = std::move(conn_pool_impl); @@ -155,7 +157,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Common::Redis::Client::Config&) override { + const Common::Redis::Client::Config&, + Common::Redis::RedisCommandStatsPtr&&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 2c9bde9dea18..8f4d4d666782 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -126,7 +126,8 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr&&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From 841b4a357a6f8af84677a4b5432f3652a20a1f13 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 5 Sep 2019 17:09:52 -0700 Subject: [PATCH 45/55] guard command timer on stats enabled Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/client_impl.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 5b8847719e97..417ee06ac34b 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -249,9 +249,11 @@ ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& ca Stats::StatName command) : parent_(parent), callbacks_(callbacks), command_{command}, aggregate_request_timer_( - parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)), - command_request_timer_( - parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_)) { + parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)) { + if (parent_.redis_command_stats_->enabled()) { + command_request_timer_ = + parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_); + } parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); From 2520b40f7bca14d64043404e227b43c06e186edf Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Thu, 5 Sep 2019 20:33:07 -0700 Subject: [PATCH 46/55] fix test override Signed-off-by: Nicolas Flacco --- test/extensions/clusters/redis/redis_cluster_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 39f23ef1be78..bf7c87b93687 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -63,7 +63,8 @@ class RedisClusterTest : public testing::Test, // ClientFactory Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr&&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; From f76c164153f517f56a64350e4a09699bff39146e Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 08:29:19 -0700 Subject: [PATCH 47/55] Update command stats ptr name and remove redundant temp Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.h | 2 +- source/extensions/filters/network/common/redis/client.h | 2 +- .../filters/network/common/redis/client_impl.cc | 6 +++--- .../extensions/filters/network/common/redis/client_impl.h | 8 ++++---- .../filters/network/common/redis/redis_command_stats.h | 8 +++----- .../filters/network/redis_proxy/conn_pool_impl.cc | 2 +- .../filters/network/redis_proxy/conn_pool_impl.h | 4 ++-- source/extensions/health_checkers/redis/redis.h | 2 +- test/extensions/clusters/redis/redis_cluster_test.cc | 2 +- .../filters/network/common/redis/client_impl_test.cc | 2 +- .../filters/network/redis_proxy/conn_pool_impl_test.cc | 2 +- test/extensions/health_checkers/redis/redis_test.cc | 2 +- 12 files changed, 20 insertions(+), 22 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 094ce5260837..bec0ac965031 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -242,7 +242,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { Event::TimerPtr resolve_timer_; NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; const std::chrono::milliseconds buffer_timeout_; - NetworkFilters::Common::Redis::RedisCommandStatsPtr redis_command_stats_; + NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; Upstream::ClusterManager& cluster_manager_; diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 21d3e20b635b..bc5e7e407f84 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -190,7 +190,7 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsPtr&& redis_command_stats) PURE; + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 417ee06ac34b..d70df1c334f4 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -49,7 +49,7 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsPtr&& redis_command_stats) { + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) { auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, config, std::move(redis_command_stats)); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; @@ -62,7 +62,7 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsPtr&& redis_command_stats) + RedisCommandStatsSharedPtr&& redis_command_stats) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() { onConnectOrOpTimeout(); })), @@ -276,7 +276,7 @@ ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, - RedisCommandStatsPtr&& redis_command_stats) { + RedisCommandStatsSharedPtr&& redis_command_stats) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, config, std::move(redis_command_stats)); } diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 72a5fe933a7e..3ff19b61d84b 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -68,11 +68,11 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsPtr&& redis_command_stats); + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats); ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsPtr&& redis_command_stats); + RedisCommandStatsSharedPtr&& redis_command_stats); ~ClientImpl() override; // Client @@ -137,14 +137,14 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne bool connected_{}; Event::TimerPtr flush_timer_; Envoy::TimeSource& time_source_; - const RedisCommandStatsPtr redis_command_stats_; + const RedisCommandStatsSharedPtr redis_command_stats_; }; class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsPtr&& redis_command_stats) override; + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 10b40d01315a..b21421aad5f6 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -21,13 +21,11 @@ class RedisCommandStats { public: RedisCommandStats(Stats::Scope&, const std::string& prefix, bool enabled); - // TODO: Make std::shared_ptr be RedisCommandStatsPtr static std::shared_ptr createRedisCommandStats(Stats::Scope& stats_scope, const std::string& prefix, bool enabled) { - auto redis_command_stats = std::make_shared( + return std::make_shared( stats_scope, prefix, - enabled); // TODO: createScope uses make_unique, should I be doing that? - return redis_command_stats; + enabled); } Stats::Counter& counter(const Stats::StatNameVec& stat_names); @@ -59,7 +57,7 @@ class RedisCommandStats { const Stats::StatName unknown_metric_; const ToLowerTable to_lower_table_; }; -using RedisCommandStatsPtr = std::shared_ptr; +using RedisCommandStatsSharedPtr = std::shared_ptr; } // namespace Redis } // namespace Common diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 7b317332808f..a045558502dc 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -24,7 +24,7 @@ InstanceImpl::InstanceImpl( Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, Api::Api& api, Stats::ScopePtr&& stats_scope, - Common::Redis::RedisCommandStatsPtr&& redis_command_stats) + Common::Redis::RedisCommandStatsSharedPtr&& redis_command_stats) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), api_(api), stats_scope_(std::move(stats_scope)), redis_command_stats_(std::move(redis_command_stats)), diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index b4a31155b9d7..0926bb1b58db 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -52,7 +52,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, Api::Api& api, Stats::ScopePtr&& stats_scope, - Common::Redis::RedisCommandStatsPtr&& redis_command_stats); + Common::Redis::RedisCommandStatsSharedPtr&& redis_command_stats); // RedisProxy::ConnPool::Instance Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, @@ -132,7 +132,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ConfigImpl config_; Api::Api& api_; Stats::ScopePtr stats_scope_; - Common::Redis::RedisCommandStatsPtr redis_command_stats_; + Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; RedisClusterStats redis_cluster_stats_; }; diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 61106195bc19..17c8e060e716 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -96,7 +96,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { RedisHealthChecker& parent_; Extensions::NetworkFilters::Common::Redis::Client::ClientPtr client_; Extensions::NetworkFilters::Common::Redis::Client::PoolRequest* current_request_{}; - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr redis_command_stats_; + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; enum class Type { Ping, Exists }; diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index bf7c87b93687..aaa93fa32351 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -64,7 +64,7 @@ class RedisClusterTest : public testing::Test, Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr&&) override { + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 9170c9cb7798..490b5b615f13 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -117,7 +117,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF ClientPtr client_; Stats::IsolatedStoreImpl stats_; Stats::ScopePtr stats_scope_; - Common::Redis::RedisCommandStatsPtr redis_command_stats_; + Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index bdf04bad137a..12bb4b7d20cf 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -158,7 +158,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&, - Common::Redis::RedisCommandStatsPtr&&) override { + Common::Redis::RedisCommandStatsSharedPtr&&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 8f4d4d666782..37c72d2b28af 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -127,7 +127,7 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsPtr&&) override { + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From 140d9e53e736a8d654484ae0156e81f4874a3fd8 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 10:41:57 -0700 Subject: [PATCH 48/55] Pass scope around, and store symboltable on command stats Signed-off-by: Nicolas Flacco --- .../clusters/redis/redis_cluster.cc | 6 ++-- .../filters/network/common/redis/client.h | 2 +- .../network/common/redis/client_impl.cc | 21 ++++++------ .../network/common/redis/client_impl.h | 7 ++-- .../common/redis/redis_command_stats.cc | 34 +++++++++---------- .../common/redis/redis_command_stats.h | 20 +++++------ .../filters/network/redis_proxy/config.cc | 9 ++--- .../network/redis_proxy/conn_pool_impl.cc | 2 +- .../clusters/redis/redis_cluster_test.cc | 2 +- .../network/common/redis/client_impl_test.cc | 10 +++--- .../redis_proxy/conn_pool_impl_test.cc | 4 +-- .../health_checkers/redis/redis_test.cc | 2 +- 12 files changed, 60 insertions(+), 59 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index f3aabc86c278..756927a224f9 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -169,8 +169,8 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( : parent_(parent), dispatcher_(parent.dispatcher_), resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), client_factory_(client_factory), buffer_timeout_(0), - redis_command_stats_(std::make_shared( - parent_.info()->statsScope(), "upstream_commands", false)) {} + redis_command_stats_(NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( + parent_.info()->statsScope().symbolTable(), "upstream_commands", false)) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. @@ -252,7 +252,7 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { client = std::make_unique(*this); client->host_ = current_host_address_; client->client_ = - client_factory_.create(host, dispatcher_, *this, std::move(redis_command_stats_)); + client_factory_.create(host, dispatcher_, *this, std::move(redis_command_stats_), parent_.info()->statsScope()); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index bc5e7e407f84..68949c28699b 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -190,7 +190,7 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) PURE; + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index d70df1c334f4..1ee8482f21f2 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -49,9 +49,9 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) { + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) { auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, - config, std::move(redis_command_stats)); + config, std::move(redis_command_stats), scope); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -62,12 +62,13 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats) + RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() { onConnectOrOpTimeout(); })), flush_timer_(dispatcher.createTimer([this]() { flushBufferAndResetTimer(); })), - time_source_(dispatcher.timeSource()), redis_command_stats_(redis_command_stats) { + time_source_(dispatcher.timeSource()), redis_command_stats_(redis_command_stats), + scope_(scope) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -100,7 +101,7 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca if (config_.enableCommandStats()) { // Only lowercase command and get StatName if we enable command stats command = redis_command_stats_->getCommandFromRequest(request); - redis_command_stats_->updateStatsTotal(command); + redis_command_stats_->updateStatsTotal(scope_, command); } else { // If disabled, we use a placeholder stat name "unused" that is not used command = redis_command_stats_->getUnusedStatName(); @@ -201,7 +202,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { if (redis_command_stats_->enabled()) { bool success = !canceled && (value->type() != Common::Redis::RespType::Error); - redis_command_stats_->updateStats(success, request.command_); + redis_command_stats_->updateStats(scope_, request.command_, success); request.command_request_timer_->complete(); } request.aggregate_request_timer_->complete(); @@ -249,10 +250,10 @@ ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& ca Stats::StatName command) : parent_(parent), callbacks_(callbacks), command_{command}, aggregate_request_timer_( - parent_.redis_command_stats_->createAggregateTimer(parent_.time_source_)) { + parent_.redis_command_stats_->createAggregateTimer(parent_.scope_, parent_.time_source_)) { if (parent_.redis_command_stats_->enabled()) { command_request_timer_ = - parent_.redis_command_stats_->createCommandTimer(command_, parent_.time_source_); + parent_.redis_command_stats_->createCommandTimer(parent_.scope_, command_, parent_.time_source_); } parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); @@ -276,9 +277,9 @@ ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats) { + RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config, std::move(redis_command_stats)); + config, std::move(redis_command_stats), scope); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 3ff19b61d84b..2ca90c3776da 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -68,11 +68,11 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats); + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope); ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats); + RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope); ~ClientImpl() override; // Client @@ -138,13 +138,14 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr flush_timer_; Envoy::TimeSource& time_source_; const RedisCommandStatsSharedPtr redis_command_stats_; + Stats::Scope& scope_; }; class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats) override; + const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 1a80f0d9869e..4c07414ce371 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,8 +8,8 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::Scope& scope, const std::string& prefix, bool enabled) - : scope_(scope), stat_name_pool_(scope.symbolTable()), prefix_(stat_name_pool_.add(prefix)), +RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled) + : symbol_table_(symbol_table), stat_name_pool_(symbol_table_), prefix_(stat_name_pool_.add(prefix)), enabled_(enabled), upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), @@ -40,28 +40,28 @@ void RedisCommandStats::addCommandToPool(const std::string& command_string) { stat_name_map_[command_string] = command; } -Stats::Counter& RedisCommandStats::counter(const Stats::StatNameVec& stat_names) { - const Stats::SymbolTable::StoragePtr storage_ptr = scope_.symbolTable().join(stat_names); +Stats::Counter& RedisCommandStats::counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); - return scope_.counterFromStatName(full_stat_name); + return scope.counterFromStatName(full_stat_name); } -Stats::Histogram& RedisCommandStats::histogram(const Stats::StatNameVec& stat_names) { - const Stats::SymbolTable::StoragePtr storage_ptr = scope_.symbolTable().join(stat_names); +Stats::Histogram& RedisCommandStats::histogram(Stats::Scope& scope, const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); - return scope_.histogramFromStatName(full_stat_name); + return scope.histogramFromStatName(full_stat_name); } Stats::CompletableTimespanPtr -RedisCommandStats::createCommandTimer(Stats::StatName command, Envoy::TimeSource& time_source) { +RedisCommandStats::createCommandTimer(Stats::Scope& scope, Stats::StatName command, Envoy::TimeSource& time_source) { return std::make_unique>( - histogram({prefix_, command, latency_}), time_source); + histogram(scope, {prefix_, command, latency_}), time_source); } Stats::CompletableTimespanPtr -RedisCommandStats::createAggregateTimer(Envoy::TimeSource& time_source) { +RedisCommandStats::createAggregateTimer(Stats::Scope& scope, Envoy::TimeSource& time_source) { return std::make_unique>( - histogram({prefix_, upstream_rq_time_}), time_source); + histogram(scope, {prefix_, upstream_rq_time_}), time_source); } Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& request) { @@ -88,15 +88,15 @@ Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& reques } } -void RedisCommandStats::updateStatsTotal(Stats::StatName command) { - counter({prefix_, command, total_}).inc(); +void RedisCommandStats::updateStatsTotal(Stats::Scope& scope, Stats::StatName command) { + counter(scope, {prefix_, command, total_}).inc(); } -void RedisCommandStats::updateStats(const bool success, Stats::StatName command) { +void RedisCommandStats::updateStats(Stats::Scope& scope, Stats::StatName command, const bool success) { if (success) { - counter({prefix_, command, success_}).inc(); + counter(scope, {prefix_, command, success_}).inc(); } else { - counter({prefix_, command, success_}).inc(); + counter(scope, {prefix_, command, success_}).inc(); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index b21421aad5f6..7a51df1d8da9 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -19,30 +19,30 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::Scope&, const std::string& prefix, bool enabled); + RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled); static std::shared_ptr - createRedisCommandStats(Stats::Scope& stats_scope, const std::string& prefix, bool enabled) { + createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled) { return std::make_shared( - stats_scope, prefix, + symbol_table, prefix, enabled); } - Stats::Counter& counter(const Stats::StatNameVec& stat_names); - Stats::Histogram& histogram(const Stats::StatNameVec& stat_names); - Stats::CompletableTimespanPtr createCommandTimer(Stats::StatName command, + Stats::Counter& counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names); + Stats::Histogram& histogram(Stats::Scope& scope, const Stats::StatNameVec& stat_names); + Stats::CompletableTimespanPtr createCommandTimer(Stats::Scope& scope, Stats::StatName command, Envoy::TimeSource& time_source); - Stats::CompletableTimespanPtr createAggregateTimer(Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createAggregateTimer(Stats::Scope& scope, Envoy::TimeSource& time_source); Stats::StatName getCommandFromRequest(const RespValue& request); - void updateStatsTotal(Stats::StatName command); - void updateStats(const bool success, Stats::StatName command); + void updateStatsTotal(Stats::Scope& scope, Stats::StatName command); + void updateStats(Stats::Scope& scope, Stats::StatName command, const bool success); bool enabled() { return enabled_; } Stats::StatName getUnusedStatName() { return unused_metric_; } private: void addCommandToPool(const std::string& command_string); - Stats::Scope& scope_; + Stats::SymbolTable& symbol_table_; Stats::StatNamePool stat_name_pool_; StringMap stat_name_map_; const Stats::StatName prefix_; diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 28e85c3f282d..b17cdb363244 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -57,18 +57,19 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP } addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); + auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( + context.scope().symbolTable(), "upstream_commands", proto_config.settings().enable_command_stats()); + Upstreams upstreams; for (auto& cluster : unique_clusters) { Stats::ScopePtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); - - auto redis_command_stats = std::make_shared( - *stats_scope, "upstream_commands", proto_config.settings().enable_command_stats()); + upstreams.emplace(cluster, std::make_shared( cluster, context.clusterManager(), Common::Redis::Client::ClientFactoryImpl::instance_, context.threadLocal(), proto_config.settings(), context.api(), - std::move(stats_scope), std::move(redis_command_stats))); + std::move(stats_scope), std::move(redis_command_stats))); // TODO: Don't use move? } auto router = diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index a045558502dc..a26a8aa69b59 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -198,7 +198,7 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared client = std::make_unique(*this); client->host_ = host; client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, - std::move(parent_.redis_command_stats_)); + std::move(parent_.redis_command_stats_), *parent_.stats_scope_); // client->redis_client_->addRedisCommandStats(parent_.redis_command_stats_); client->redis_client_->addConnectionCallbacks(*client); // TODO(hyang): should the auth command and readonly command be moved to the factory method? diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index aaa93fa32351..ce0fd32abea1 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -64,7 +64,7 @@ class RedisClusterTest : public testing::Test, Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&) override { + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope&) { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 490b5b615f13..3b10d540ed37 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -73,12 +73,11 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, connect()); EXPECT_CALL(*upstream_connection_, noDelay(true)); - stats_scope_ = stats_.createScope("test"); redis_command_stats_ = Common::Redis::RedisCommandStats::createRedisCommandStats( - *stats_scope_, "upstream_commands", true); + stats_.symbolTable(), "upstream_commands", true); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_, std::move(redis_command_stats_)); + *config_, std::move(redis_command_stats_), stats_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -885,10 +884,9 @@ TEST(RedisClientFactoryImplTest, Basic) { NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); Stats::IsolatedStoreImpl stats_; - Stats::ScopePtr stats_scope = stats_.createScope("test"); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - *stats_scope, "upstream_commands", true); - ClientPtr client = factory.create(host, dispatcher, config, std::move(redis_command_stats)); + stats_.symbolTable(), "upstream_commands", true); + ClientPtr client = factory.create(host, dispatcher, config, std::move(redis_command_stats), stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 12bb4b7d20cf..d1e77dd37161 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -67,7 +67,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client })); auto redis_command_stats = - std::make_shared(*store, "upstream_commands", true); + Common::Redis::RedisCommandStats::createRedisCommandStats(store->symbolTable(), "upstream_commands", true); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( @@ -158,7 +158,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&, - Common::Redis::RedisCommandStatsSharedPtr&&) override { + Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 37c72d2b28af..f5353ffcbb61 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -127,7 +127,7 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&) override { + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope& scope) { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From 414e5b39f3df3f0208a500a2ec69aec988efba6a Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 11:12:55 -0700 Subject: [PATCH 49/55] Remove the move part Signed-off-by: Nicolas Flacco --- .../clusters/redis/redis_cluster.cc | 9 ++++---- .../filters/network/common/redis/client.h | 4 +++- .../network/common/redis/client_impl.cc | 21 +++++++++++-------- .../network/common/redis/client_impl.h | 9 +++++--- .../common/redis/redis_command_stats.cc | 20 +++++++++++------- .../common/redis/redis_command_stats.h | 10 ++++----- .../filters/network/redis_proxy/config.cc | 16 +++++++------- .../network/redis_proxy/conn_pool_impl.cc | 11 +++++----- .../network/redis_proxy/conn_pool_impl.h | 2 +- .../extensions/health_checkers/redis/redis.cc | 12 +++++------ .../clusters/redis/redis_cluster_test.cc | 3 ++- .../network/common/redis/client_impl_test.cc | 3 ++- .../redis_proxy/conn_pool_impl_test.cc | 7 ++++--- .../health_checkers/redis/redis_test.cc | 3 ++- 14 files changed, 75 insertions(+), 55 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 756927a224f9..cefaf1c82112 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -169,8 +169,9 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( : parent_(parent), dispatcher_(parent.dispatcher_), resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), client_factory_(client_factory), buffer_timeout_(0), - redis_command_stats_(NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( - parent_.info()->statsScope().symbolTable(), "upstream_commands", false)) {} + redis_command_stats_( + NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( + parent_.info()->statsScope().symbolTable(), "upstream_commands", false)) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. @@ -251,8 +252,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = - client_factory_.create(host, dispatcher_, *this, std::move(redis_command_stats_), parent_.info()->statsScope()); + client->client_ = client_factory_.create( + host, dispatcher_, *this, std::move(redis_command_stats_), parent_.info()->statsScope()); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 68949c28699b..3e5d80ce9208 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -190,7 +190,9 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) PURE; + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 1ee8482f21f2..4a547b13fb33 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -49,9 +49,11 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) { + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) { auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, - config, std::move(redis_command_stats), scope); + config, redis_command_stats, scope); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -62,7 +64,7 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) + const RedisCommandStatsSharedPtr& redis_command_stats, Stats::Scope& scope) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), connect_or_op_timer_(dispatcher.createTimer([this]() { onConnectOrOpTimeout(); })), @@ -249,11 +251,11 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, Stats::StatName command) : parent_(parent), callbacks_(callbacks), command_{command}, - aggregate_request_timer_( - parent_.redis_command_stats_->createAggregateTimer(parent_.scope_, parent_.time_source_)) { + aggregate_request_timer_(parent_.redis_command_stats_->createAggregateTimer( + parent_.scope_, parent_.time_source_)) { if (parent_.redis_command_stats_->enabled()) { - command_request_timer_ = - parent_.redis_command_stats_->createCommandTimer(parent_.scope_, command_, parent_.time_source_); + command_request_timer_ = parent_.redis_command_stats_->createCommandTimer( + parent_.scope_, command_, parent_.time_source_); } parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); @@ -277,9 +279,10 @@ ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) { + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config, std::move(redis_command_stats), scope); + config, redis_command_stats, scope); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 2ca90c3776da..a8ab3806eb7e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -68,11 +68,13 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope); + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope); ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, - RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope); + const RedisCommandStatsSharedPtr& redis_command_stats, Stats::Scope& scope); ~ClientImpl() override; // Client @@ -145,7 +147,8 @@ class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config, RedisCommandStatsSharedPtr&& redis_command_stats, Stats::Scope& scope) override; + const Config& config, const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 4c07414ce371..5b2cd3f19b9c 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,9 +8,11 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled) - : symbol_table_(symbol_table), stat_name_pool_(symbol_table_), prefix_(stat_name_pool_.add(prefix)), - enabled_(enabled), upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), +RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, + bool enabled) + : symbol_table_(symbol_table), stat_name_pool_(symbol_table_), + prefix_(stat_name_pool_.add(prefix)), enabled_(enabled), + upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), unused_metric_(stat_name_pool_.add("unused")), null_metric_(stat_name_pool_.add("null")), @@ -40,20 +42,23 @@ void RedisCommandStats::addCommandToPool(const std::string& command_string) { stat_name_map_[command_string] = command; } -Stats::Counter& RedisCommandStats::counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names) { +Stats::Counter& RedisCommandStats::counter(Stats::Scope& scope, + const Stats::StatNameVec& stat_names) { const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); return scope.counterFromStatName(full_stat_name); } -Stats::Histogram& RedisCommandStats::histogram(Stats::Scope& scope, const Stats::StatNameVec& stat_names) { +Stats::Histogram& RedisCommandStats::histogram(Stats::Scope& scope, + const Stats::StatNameVec& stat_names) { const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); return scope.histogramFromStatName(full_stat_name); } Stats::CompletableTimespanPtr -RedisCommandStats::createCommandTimer(Stats::Scope& scope, Stats::StatName command, Envoy::TimeSource& time_source) { +RedisCommandStats::createCommandTimer(Stats::Scope& scope, Stats::StatName command, + Envoy::TimeSource& time_source) { return std::make_unique>( histogram(scope, {prefix_, command, latency_}), time_source); } @@ -92,7 +97,8 @@ void RedisCommandStats::updateStatsTotal(Stats::Scope& scope, Stats::StatName co counter(scope, {prefix_, command, total_}).inc(); } -void RedisCommandStats::updateStats(Stats::Scope& scope, Stats::StatName command, const bool success) { +void RedisCommandStats::updateStats(Stats::Scope& scope, Stats::StatName command, + const bool success) { if (success) { counter(scope, {prefix_, command, success_}).inc(); } else { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 7a51df1d8da9..87b9992bc692 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -22,17 +22,17 @@ class RedisCommandStats { RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled); static std::shared_ptr - createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled) { - return std::make_shared( - symbol_table, prefix, - enabled); + createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, + bool enabled) { + return std::make_shared(symbol_table, prefix, enabled); } Stats::Counter& counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names); Stats::Histogram& histogram(Stats::Scope& scope, const Stats::StatNameVec& stat_names); Stats::CompletableTimespanPtr createCommandTimer(Stats::Scope& scope, Stats::StatName command, Envoy::TimeSource& time_source); - Stats::CompletableTimespanPtr createAggregateTimer(Stats::Scope& scope, Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createAggregateTimer(Stats::Scope& scope, + Envoy::TimeSource& time_source); Stats::StatName getCommandFromRequest(const RespValue& request); void updateStatsTotal(Stats::Scope& scope, Stats::StatName command); void updateStats(Stats::Scope& scope, Stats::StatName command, const bool success); diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index b17cdb363244..ba8877798175 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -58,18 +58,20 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - context.scope().symbolTable(), "upstream_commands", proto_config.settings().enable_command_stats()); + context.scope().symbolTable(), "upstream_commands", + proto_config.settings().enable_command_stats()); Upstreams upstreams; for (auto& cluster : unique_clusters) { Stats::ScopePtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); - - upstreams.emplace(cluster, std::make_shared( - cluster, context.clusterManager(), - Common::Redis::Client::ClientFactoryImpl::instance_, - context.threadLocal(), proto_config.settings(), context.api(), - std::move(stats_scope), std::move(redis_command_stats))); // TODO: Don't use move? + + upstreams.emplace(cluster, + std::make_shared( + cluster, context.clusterManager(), + Common::Redis::Client::ClientFactoryImpl::instance_, + context.threadLocal(), proto_config.settings(), context.api(), + std::move(stats_scope), redis_command_stats)); // TODO: Don't use move? } auto router = diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index a26a8aa69b59..c8449f61dc84 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -24,11 +24,11 @@ InstanceImpl::InstanceImpl( Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, Api::Api& api, Stats::ScopePtr&& stats_scope, - Common::Redis::RedisCommandStatsSharedPtr&& redis_command_stats) + const Common::Redis::RedisCommandStatsSharedPtr& redis_command_stats) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), api_(api), stats_scope_(std::move(stats_scope)), - redis_command_stats_(std::move(redis_command_stats)), - redis_cluster_stats_{REDIS_CLUSTER_STATS(POOL_COUNTER(*stats_scope_))} { + redis_command_stats_(redis_command_stats), redis_cluster_stats_{REDIS_CLUSTER_STATS( + POOL_COUNTER(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -197,9 +197,8 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, - std::move(parent_.redis_command_stats_), *parent_.stats_scope_); - // client->redis_client_->addRedisCommandStats(parent_.redis_command_stats_); + client->redis_client_ = parent_.client_factory_.create( + host, dispatcher_, parent_.config_, parent_.redis_command_stats_, *parent_.stats_scope_); client->redis_client_->addConnectionCallbacks(*client); // TODO(hyang): should the auth command and readonly command be moved to the factory method? if (!auth_password_.empty()) { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 0926bb1b58db..7731943b873f 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -52,7 +52,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, Api::Api& api, Stats::ScopePtr&& stats_scope, - Common::Redis::RedisCommandStatsSharedPtr&& redis_command_stats); + const Common::Redis::RedisCommandStatsSharedPtr& redis_command_stats); // RedisProxy::ConnPool::Instance Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index acc6204e8fa5..81fe769a5cd7 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -23,10 +23,9 @@ RedisHealthChecker::RedisHealthChecker( RedisHealthChecker::RedisActiveHealthCheckSession::RedisActiveHealthCheckSession( RedisHealthChecker& parent, const Upstream::HostSharedPtr& host) : ActiveHealthCheckSession(parent, host), parent_(parent) { - auto redis_command_stats = - std::make_shared( - parent_.cluster_.info()->statsScope(), "upstream_commands", false); - redis_command_stats_ = std::move(redis_command_stats); + redis_command_stats_ = + Extensions::NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( + parent_.cluster_.info()->statsScope().symbolTable(), "upstream_commands", false); } RedisHealthChecker::RedisActiveHealthCheckSession::~RedisActiveHealthCheckSession() { @@ -56,8 +55,9 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this, - std::move(redis_command_stats_)); + client_ = + parent_.client_factory_.create(host_, parent_.dispatcher_, *this, redis_command_stats_, + parent_.cluster_.info()->statsScope()); client_->addConnectionCallbacks(*this); } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index ce0fd32abea1..74de8db36483 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -64,7 +64,8 @@ class RedisClusterTest : public testing::Test, Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope&) { + const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 3b10d540ed37..05c14e51838c 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -886,7 +886,8 @@ TEST(RedisClientFactoryImplTest, Basic) { Stats::IsolatedStoreImpl stats_; auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( stats_.symbolTable(), "upstream_commands", true); - ClientPtr client = factory.create(host, dispatcher, config, std::move(redis_command_stats), stats_); + ClientPtr client = + factory.create(host, dispatcher, config, std::move(redis_command_stats), stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index d1e77dd37161..10059590fc61 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -66,8 +66,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); - auto redis_command_stats = - Common::Redis::RedisCommandStats::createRedisCommandStats(store->symbolTable(), "upstream_commands", true); + auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( + store->symbolTable(), "upstream_commands", true); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( @@ -158,7 +158,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&, - Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope&) override { + const Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index f5353ffcbb61..7ce0e8c82821 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -127,7 +127,8 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, - Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&&, Stats::Scope& scope) { + const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From 9ec13ee0bec915cb28b113dd24be4d79621cc9cf Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 11:45:08 -0700 Subject: [PATCH 50/55] refactor boolean enable flag, since now a global redis stats Signed-off-by: Nicolas Flacco --- .../clusters/redis/redis_cluster.cc | 2 +- .../network/common/redis/client_impl.cc | 4 +-- .../common/redis/redis_command_stats.cc | 35 +++++++++---------- .../common/redis/redis_command_stats.h | 9 ++--- .../filters/network/redis_proxy/config.cc | 14 ++++---- .../extensions/health_checkers/redis/redis.cc | 2 +- .../network/common/redis/client_impl_test.cc | 4 +-- .../redis_proxy/conn_pool_impl_test.cc | 2 +- 8 files changed, 32 insertions(+), 40 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index cefaf1c82112..4b89102636f0 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -171,7 +171,7 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( client_factory_(client_factory), buffer_timeout_(0), redis_command_stats_( NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( - parent_.info()->statsScope().symbolTable(), "upstream_commands", false)) {} + parent_.info()->statsScope().symbolTable(), "upstream_commands")) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 4a547b13fb33..acb7d64f771e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -202,7 +202,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { PendingRequest& request = pending_requests_.front(); const bool canceled = request.canceled_; - if (redis_command_stats_->enabled()) { + if (config_.enableCommandStats()) { bool success = !canceled && (value->type() != Common::Redis::RespType::Error); redis_command_stats_->updateStats(scope_, request.command_, success); request.command_request_timer_->complete(); @@ -253,7 +253,7 @@ ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& ca : parent_(parent), callbacks_(callbacks), command_{command}, aggregate_request_timer_(parent_.redis_command_stats_->createAggregateTimer( parent_.scope_, parent_.time_source_)) { - if (parent_.redis_command_stats_->enabled()) { + if (parent_.config_.enableCommandStats()) { command_request_timer_ = parent_.redis_command_stats_->createCommandTimer( parent_.scope_, command_, parent_.time_source_); } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 5b2cd3f19b9c..ce11df704bac 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -8,33 +8,30 @@ namespace NetworkFilters { namespace Common { namespace Redis { -RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, - bool enabled) +RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix) : symbol_table_(symbol_table), stat_name_pool_(symbol_table_), - prefix_(stat_name_pool_.add(prefix)), enabled_(enabled), + prefix_(stat_name_pool_.add(prefix)), upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), unused_metric_(stat_name_pool_.add("unused")), null_metric_(stat_name_pool_.add("null")), unknown_metric_(stat_name_pool_.add("unknown")) { // Note: Even if this is disabled, we track the upstream_rq_time. - if (enabled_) { - // Create StatName for each Redis command. Note that we don't include Auth or Ping. - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { - addCommandToPool(command); - } - for (const std::string& command : - Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { - addCommandToPool(command); - } - for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: - hashMultipleSumResultCommands()) { - addCommandToPool(command); - } - addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); + // Create StatName for each Redis command. Note that we don't include Auth or Ping. + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + addCommandToPool(command); + } + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + addCommandToPool(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()) { + addCommandToPool(command); } + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } void RedisCommandStats::addCommandToPool(const std::string& command_string) { diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 87b9992bc692..79e39c8d80e2 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -19,12 +19,11 @@ namespace Redis { class RedisCommandStats { public: - RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, bool enabled); + RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix); static std::shared_ptr - createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix, - bool enabled) { - return std::make_shared(symbol_table, prefix, enabled); + createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix) { + return std::make_shared(symbol_table, prefix); } Stats::Counter& counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names); @@ -36,7 +35,6 @@ class RedisCommandStats { Stats::StatName getCommandFromRequest(const RespValue& request); void updateStatsTotal(Stats::Scope& scope, Stats::StatName command); void updateStats(Stats::Scope& scope, Stats::StatName command, const bool success); - bool enabled() { return enabled_; } Stats::StatName getUnusedStatName() { return unused_metric_; } private: @@ -46,7 +44,6 @@ class RedisCommandStats { Stats::StatNamePool stat_name_pool_; StringMap stat_name_map_; const Stats::StatName prefix_; - bool enabled_; const Stats::StatName upstream_rq_time_; const Stats::StatName latency_; const Stats::StatName total_; diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index ba8877798175..315e41aafbd2 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -58,20 +58,18 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - context.scope().symbolTable(), "upstream_commands", - proto_config.settings().enable_command_stats()); + context.scope().symbolTable(), "upstream_commands"); Upstreams upstreams; for (auto& cluster : unique_clusters) { Stats::ScopePtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); - upstreams.emplace(cluster, - std::make_shared( - cluster, context.clusterManager(), - Common::Redis::Client::ClientFactoryImpl::instance_, - context.threadLocal(), proto_config.settings(), context.api(), - std::move(stats_scope), redis_command_stats)); // TODO: Don't use move? + upstreams.emplace(cluster, std::make_shared( + cluster, context.clusterManager(), + Common::Redis::Client::ClientFactoryImpl::instance_, + context.threadLocal(), proto_config.settings(), context.api(), + std::move(stats_scope), redis_command_stats)); } auto router = diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 81fe769a5cd7..93edabd219da 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -25,7 +25,7 @@ RedisHealthChecker::RedisActiveHealthCheckSession::RedisActiveHealthCheckSession : ActiveHealthCheckSession(parent, host), parent_(parent) { redis_command_stats_ = Extensions::NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( - parent_.cluster_.info()->statsScope().symbolTable(), "upstream_commands", false); + parent_.cluster_.info()->statsScope().symbolTable(), "upstream_commands"); } RedisHealthChecker::RedisActiveHealthCheckSession::~RedisActiveHealthCheckSession() { diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 05c14e51838c..c1dcb05ab7a5 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -74,7 +74,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, noDelay(true)); redis_command_stats_ = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable(), "upstream_commands", true); + stats_.symbolTable(), "upstream_commands"); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, *config_, std::move(redis_command_stats_), stats_); @@ -885,7 +885,7 @@ TEST(RedisClientFactoryImplTest, Basic) { ConfigImpl config(createConnPoolSettings()); Stats::IsolatedStoreImpl stats_; auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable(), "upstream_commands", true); + stats_.symbolTable(), "upstream_commands"); ClientPtr client = factory.create(host, dispatcher, config, std::move(redis_command_stats), stats_); client->close(); diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 10059590fc61..4f78b8a5c625 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -67,7 +67,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client })); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - store->symbolTable(), "upstream_commands", true); + store->symbolTable(), "upstream_commands"); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( From 4ad0e2c8f4ce9fb5773837064a127171b9800706 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 13:32:48 -0700 Subject: [PATCH 51/55] remove some unneeded moves Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.cc | 4 ++-- .../filters/network/common/redis/client_impl_test.cc | 5 ++--- .../filters/network/redis_proxy/conn_pool_impl_test.cc | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 4b89102636f0..781f9844d362 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -252,8 +252,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = client_factory_.create( - host, dispatcher_, *this, std::move(redis_command_stats_), parent_.info()->statsScope()); + client->client_ = client_factory_.create(host, dispatcher_, *this, redis_command_stats_, + parent_.info()->statsScope()); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index c1dcb05ab7a5..2919243a9b28 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -77,7 +77,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF stats_.symbolTable(), "upstream_commands"); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_, std::move(redis_command_stats_), stats_); + *config_, redis_command_stats_, stats_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -886,8 +886,7 @@ TEST(RedisClientFactoryImplTest, Basic) { Stats::IsolatedStoreImpl stats_; auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( stats_.symbolTable(), "upstream_commands"); - ClientPtr client = - factory.create(host, dispatcher, config, std::move(redis_command_stats), stats_); + ClientPtr client = factory.create(host, dispatcher, config, redis_command_stats, stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 4f78b8a5c625..90175869005e 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -72,7 +72,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( 20, hashtagging, true, max_unknown_conns, read_policy_), - api_, std::move(store), std::move(redis_command_stats)); + api_, std::move(store), redis_command_stats); // Set the authentication password for this connection pool. conn_pool_impl->tls_->getTyped().auth_password_ = auth_password_; conn_pool_ = std::move(conn_pool_impl); From 228af548ca4a8a283591bbeb04fd133229bef4cc Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Fri, 6 Sep 2019 15:03:02 -0700 Subject: [PATCH 52/55] fix clang tidy errors Signed-off-by: Nicolas Flacco --- test/extensions/clusters/redis/redis_cluster_test.cc | 2 +- test/extensions/health_checkers/redis/redis_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 74de8db36483..3a0146e668f0 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -65,7 +65,7 @@ class RedisClusterTest : public testing::Test, create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, - Stats::Scope&) { + Stats::Scope&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 7ce0e8c82821..a669e1a41acc 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -128,7 +128,7 @@ class RedisHealthCheckerTest create(Upstream::HostConstSharedPtr, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, - Stats::Scope&) { + Stats::Scope&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } From d578a4bae31196c00c2a190af35b22fa1c787307 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 10 Sep 2019 09:51:13 -0700 Subject: [PATCH 53/55] Add TODO comment for singleton and remove duplicate params Signed-off-by: Nicolas Flacco --- source/extensions/clusters/redis/redis_cluster.cc | 2 +- .../filters/network/common/redis/redis_command_stats.h | 5 +++-- source/extensions/filters/network/redis_proxy/config.cc | 2 +- source/extensions/health_checkers/redis/redis.cc | 2 +- .../filters/network/common/redis/client_impl_test.cc | 4 ++-- .../filters/network/redis_proxy/conn_pool_impl_test.cc | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 781f9844d362..cf6f1d9f5bad 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -171,7 +171,7 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( client_factory_(client_factory), buffer_timeout_(0), redis_command_stats_( NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( - parent_.info()->statsScope().symbolTable(), "upstream_commands")) {} + parent_.info()->statsScope().symbolTable())) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index 79e39c8d80e2..f459f32278b5 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -21,9 +21,10 @@ class RedisCommandStats { public: RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix); + // TODO (@FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg): Use Singleton to manage a single RedisCommandStats on the client factory so that it can be used for proxy filter, discovery and health check. static std::shared_ptr - createRedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix) { - return std::make_shared(symbol_table, prefix); + createRedisCommandStats(Stats::SymbolTable& symbol_table) { + return std::make_shared(symbol_table, "upstream_commands"); } Stats::Counter& counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names); diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 315e41aafbd2..e47d7f3d2553 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -58,7 +58,7 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - context.scope().symbolTable(), "upstream_commands"); + context.scope().symbolTable()); Upstreams upstreams; for (auto& cluster : unique_clusters) { diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 93edabd219da..c1107e1a489f 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -25,7 +25,7 @@ RedisHealthChecker::RedisActiveHealthCheckSession::RedisActiveHealthCheckSession : ActiveHealthCheckSession(parent, host), parent_(parent) { redis_command_stats_ = Extensions::NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( - parent_.cluster_.info()->statsScope().symbolTable(), "upstream_commands"); + parent_.cluster_.info()->statsScope().symbolTable()); } RedisHealthChecker::RedisActiveHealthCheckSession::~RedisActiveHealthCheckSession() { diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 13079a7df1f3..bda7621b29d1 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -70,7 +70,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, noDelay(true)); redis_command_stats_ = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable(), "upstream_commands"); + stats_.symbolTable()); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, *config_, redis_command_stats_, stats_); @@ -881,7 +881,7 @@ TEST(RedisClientFactoryImplTest, Basic) { ConfigImpl config(createConnPoolSettings()); Stats::IsolatedStoreImpl stats_; auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable(), "upstream_commands"); + stats_.symbolTable()); ClientPtr client = factory.create(host, dispatcher, config, redis_command_stats, stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 90175869005e..8b796aa1b13a 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -67,7 +67,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client })); auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - store->symbolTable(), "upstream_commands"); + store->symbolTable()); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( From 6904da0edde483af12947cba1ece97eba2251c5d Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 10 Sep 2019 09:59:19 -0700 Subject: [PATCH 54/55] fix format Signed-off-by: Nicolas Flacco --- .../filters/network/common/redis/redis_command_stats.h | 4 +++- source/extensions/filters/network/redis_proxy/config.cc | 4 ++-- .../filters/network/common/redis/client_impl_test.cc | 8 ++++---- .../filters/network/redis_proxy/conn_pool_impl_test.cc | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index f459f32278b5..0ee2ce824c48 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -21,7 +21,9 @@ class RedisCommandStats { public: RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix); - // TODO (@FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg): Use Singleton to manage a single RedisCommandStats on the client factory so that it can be used for proxy filter, discovery and health check. + // TODO (@FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg): Use Singleton to manage a single + // RedisCommandStats on the client factory so that it can be used for proxy filter, discovery and + // health check. static std::shared_ptr createRedisCommandStats(Stats::SymbolTable& symbol_table) { return std::make_shared(symbol_table, "upstream_commands"); diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index e47d7f3d2553..f14176ebe274 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -57,8 +57,8 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP } addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); - auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - context.scope().symbolTable()); + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(context.scope().symbolTable()); Upstreams upstreams; for (auto& cluster : unique_clusters) { diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index bda7621b29d1..8a283c67faf1 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -69,8 +69,8 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, connect()); EXPECT_CALL(*upstream_connection_, noDelay(true)); - redis_command_stats_ = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable()); + redis_command_stats_ = + Common::Redis::RedisCommandStats::createRedisCommandStats(stats_.symbolTable()); client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, *config_, redis_command_stats_, stats_); @@ -880,8 +880,8 @@ TEST(RedisClientFactoryImplTest, Basic) { NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); Stats::IsolatedStoreImpl stats_; - auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - stats_.symbolTable()); + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(stats_.symbolTable()); ClientPtr client = factory.create(host, dispatcher, config, redis_command_stats, stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 8b796aa1b13a..f8f03045aa8e 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -66,8 +66,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); - auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats( - store->symbolTable()); + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(store->symbolTable()); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( From 4fe92657a8afeaa703999faea680faab29beb09f Mon Sep 17 00:00:00 2001 From: Nicolas Flacco Date: Tue, 10 Sep 2019 13:15:09 -0700 Subject: [PATCH 55/55] Update per mklein comments Signed-off-by: Nicolas Flacco --- docs/root/intro/arch_overview/other_protocols/redis.rst | 4 ++++ docs/root/intro/version_history.rst | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 980cc603217e..e96cbe6a3e58 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -60,6 +60,8 @@ If passive healthchecking is desired, also configure For the purposes of passive healthchecking, connect timeouts, command timeouts, and connection close map to 5xx. All other responses from Redis are counted as a success. +.. _arch_overview_redis_cluster_support: + Redis Cluster Support (Experimental) ---------------------------------------- @@ -92,6 +94,8 @@ Every Redis cluster has its own extra statistics tree rooted at *cluster.. upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed upstream_commands.upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests +.. _arch_overview_redis_cluster_command_stats: + Per-cluster command statistics can be enabled via the setting :ref:`enable_command_stats `: .. csv-table:: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 45cfc715a9f9..679ea4f9e769 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -33,14 +33,12 @@ Version history * http: remove h2c upgrade headers for HTTP/1 as h2c upgrades are currently not supported. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. -* redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. -* redis: added :ref:`enable_command_stats ` to enable per command statistics for upstream clusters. -* rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * outlier_detector: added :ref:`support for the grpc-status response header ` by mapping it to HTTP status. Guarded by envoy.reloadable_features.outlier_detection_support_for_grpc_status which defaults to true. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). * rbac: added support for DNS SAN as :ref:`principal_name `. +* redis: added :ref:`enable_command_stats ` to enable :ref:`per command statistics ` for upstream clusters. * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. * regex: introduce new :ref:`RegexMatcher ` type that provides a safe regex implementation for untrusted user input. This type is now used in all