From 9646b908a75de403aa460312593b612456808aac Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Fri, 19 Jul 2024 10:13:45 +0300 Subject: [PATCH] feat grpc: add GenericClient 45c630765c628f16cd600ab26711d67e118c8767 --- .mapping.json | 8 ++ core/include/userver/concurrent/variable.hpp | 6 + .../utils/statistics/histogram_view.hpp | 8 +- .../userver/utils/statistics/metric_value.hpp | 28 +++- .../utils/statistics/striped_rate_counter.hpp | 2 +- core/src/utils/statistics/histogram_view.cpp | 5 - .../statistics/impl/histogram_view_utils.hpp | 3 +- .../userver/utils/statistics/testing.hpp | 1 + .../userver/ugrpc/byte_buffer_utils.hpp | 38 ++++++ .../userver/ugrpc/client/client_factory.hpp | 23 ++-- grpc/include/userver/ugrpc/client/generic.hpp | 88 ++++++++++++ .../ugrpc/client/impl/async_methods.hpp | 24 +--- .../userver/ugrpc/client/impl/call_params.hpp | 32 ++--- .../userver/ugrpc/client/impl/client_data.hpp | 55 +++++--- .../userver/ugrpc/client/middlewares/base.hpp | 7 + .../client/middlewares/log/component.hpp | 8 +- grpc/include/userver/ugrpc/client/rpc.hpp | 80 ++++++----- .../userver/ugrpc/impl/maybe_owned_string.hpp | 47 +++++++ .../include/userver/ugrpc/impl/statistics.hpp | 16 ++- .../userver/ugrpc/impl/statistics_storage.hpp | 70 +++++++--- grpc/src/ugrpc/byte_buffer_utils.cpp | 40 ++++++ grpc/src/ugrpc/client/generic.cpp | 48 +++++++ grpc/src/ugrpc/client/impl/async_methods.cpp | 11 +- grpc/src/ugrpc/client/impl/call_params.cpp | 78 +++++++++-- grpc/src/ugrpc/client/impl/client_data.cpp | 34 +++++ grpc/src/ugrpc/client/middlewares/base.cpp | 14 ++ .../client/middlewares/log/component.cpp | 19 ++- .../client/middlewares/log/middleware.cpp | 2 +- .../client/middlewares/log/middleware.hpp | 19 ++- grpc/src/ugrpc/impl/statistics.cpp | 19 ++- grpc/src/ugrpc/impl/statistics_storage.cpp | 122 ++++++++++++++--- grpc/tests/src/base_test.cpp | 1 + grpc/tests/src/generic_client_test.cpp | 125 ++++++++++++++++++ grpc/tests/src/logging_test.cpp | 2 +- grpc/utest/src/ugrpc/tests/service.cpp | 3 +- scripts/grpc/templates/client.usrv.cpp.jinja | 4 +- scripts/grpc/templates/client.usrv.hpp.jinja | 2 +- .../userver/utils/impl/source_location.hpp | 2 + universal/include/userver/utils/span.hpp | 22 ++- universal/src/utils/impl/source_location.cpp | 16 +++ .../userver/utest/log_capture_fixture.hpp | 32 ++++- .../utest/src/utest/log_capture_fixture.cpp | 50 ++++--- ydb/tests/small_table.hpp | 7 +- 43 files changed, 970 insertions(+), 251 deletions(-) create mode 100644 grpc/include/userver/ugrpc/byte_buffer_utils.hpp create mode 100644 grpc/include/userver/ugrpc/client/generic.hpp create mode 100644 grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp create mode 100644 grpc/src/ugrpc/byte_buffer_utils.cpp create mode 100644 grpc/src/ugrpc/client/generic.cpp create mode 100644 grpc/src/ugrpc/client/impl/client_data.cpp create mode 100644 grpc/tests/src/generic_client_test.cpp create mode 100644 universal/src/utils/impl/source_location.cpp diff --git a/.mapping.json b/.mapping.json index e0af039c72bf..e056b493e9b1 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1771,11 +1771,13 @@ "grpc/handlers/proto/healthchecking/healthchecking.proto":"taxi/uservices/userver/grpc/handlers/proto/healthchecking/healthchecking.proto", "grpc/handlers/src/ugrpc/server/health/health.cpp":"taxi/uservices/userver/grpc/handlers/src/ugrpc/server/health/health.cpp", "grpc/handlers/src/ugrpc/server/health/test_test.cpp":"taxi/uservices/userver/grpc/handlers/src/ugrpc/server/health/test_test.cpp", + "grpc/include/userver/ugrpc/byte_buffer_utils.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/byte_buffer_utils.hpp", "grpc/include/userver/ugrpc/client/channels.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/channels.hpp", "grpc/include/userver/ugrpc/client/client_factory.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory.hpp", "grpc/include/userver/ugrpc/client/client_factory_component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory_component.hpp", "grpc/include/userver/ugrpc/client/exceptions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/exceptions.hpp", "grpc/include/userver/ugrpc/client/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/fwd.hpp", + "grpc/include/userver/ugrpc/client/generic.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/generic.hpp", "grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/client/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_methods.hpp", "grpc/include/userver/ugrpc/client/impl/call_params.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_params.hpp", @@ -1797,6 +1799,7 @@ "grpc/include/userver/ugrpc/impl/completion_queues.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/completion_queues.hpp", "grpc/include/userver/ugrpc/impl/deadline_timepoint.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/deadline_timepoint.hpp", "grpc/include/userver/ugrpc/impl/internal_tag_fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/internal_tag_fwd.hpp", + "grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp", "grpc/include/userver/ugrpc/impl/queue_runner.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/queue_runner.hpp", "grpc/include/userver/ugrpc/impl/span.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/span.hpp", "grpc/include/userver/ugrpc/impl/static_metadata.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/static_metadata.hpp", @@ -1835,16 +1838,19 @@ "grpc/proto/tests/repeating_word_in_package_name.proto":"taxi/uservices/userver/grpc/proto/tests/repeating_word_in_package_name.proto", "grpc/proto/tests/same_service_and_method_name.proto":"taxi/uservices/userver/grpc/proto/tests/same_service_and_method_name.proto", "grpc/proto/tests/unit_test.proto":"taxi/uservices/userver/grpc/proto/tests/unit_test.proto", + "grpc/src/ugrpc/byte_buffer_utils.cpp":"taxi/uservices/userver/grpc/src/ugrpc/byte_buffer_utils.cpp", "grpc/src/ugrpc/client/channels.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/channels.cpp", "grpc/src/ugrpc/client/client_factory.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/client_factory.cpp", "grpc/src/ugrpc/client/client_factory_component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/client_factory_component.cpp", "grpc/src/ugrpc/client/exceptions.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/exceptions.cpp", + "grpc/src/ugrpc/client/generic.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/generic.cpp", "grpc/src/ugrpc/client/impl/async_method_invocation.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_method_invocation.cpp", "grpc/src/ugrpc/client/impl/async_methods.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_methods.cpp", "grpc/src/ugrpc/client/impl/call_params.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_params.cpp", "grpc/src/ugrpc/client/impl/channel_cache.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/channel_cache.cpp", "grpc/src/ugrpc/client/impl/client_configs.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_configs.cpp", "grpc/src/ugrpc/client/impl/client_configs.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_configs.hpp", + "grpc/src/ugrpc/client/impl/client_data.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_data.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.hpp", "grpc/src/ugrpc/client/impl/client_qos.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_qos.cpp", @@ -1920,6 +1926,7 @@ "grpc/tests/src/deadline_metrics_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_metrics_test.cpp", "grpc/tests/src/deadline_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_test.cpp", "grpc/tests/src/error_test.cpp":"taxi/uservices/userver/grpc/tests/src/error_test.cpp", + "grpc/tests/src/generic_client_test.cpp":"taxi/uservices/userver/grpc/tests/src/generic_client_test.cpp", "grpc/tests/src/logging_test.cpp":"taxi/uservices/userver/grpc/tests/src/logging_test.cpp", "grpc/tests/src/middlewares_test.cpp":"taxi/uservices/userver/grpc/tests/src/middlewares_test.cpp", "grpc/tests/src/serialization_test.cpp":"taxi/uservices/userver/grpc/tests/src/serialization_test.cpp", @@ -4024,6 +4031,7 @@ "universal/src/utils/impl/disable_core_dumps.cpp":"taxi/uservices/userver/universal/src/utils/impl/disable_core_dumps.cpp", "universal/src/utils/impl/internal_tag.hpp":"taxi/uservices/userver/universal/src/utils/impl/internal_tag.hpp", "universal/src/utils/impl/projecting_view_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/projecting_view_test.cpp", + "universal/src/utils/impl/source_location.cpp":"taxi/uservices/userver/universal/src/utils/impl/source_location.cpp", "universal/src/utils/impl/source_location_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/source_location_test.cpp", "universal/src/utils/impl/static_registration.cpp":"taxi/uservices/userver/universal/src/utils/impl/static_registration.cpp", "universal/src/utils/impl/transparent_hash_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/transparent_hash_test.cpp", diff --git a/core/include/userver/concurrent/variable.hpp b/core/include/userver/concurrent/variable.hpp index 4a5e75505e15..24c9d7b1966d 100644 --- a/core/include/userver/concurrent/variable.hpp +++ b/core/include/userver/concurrent/variable.hpp @@ -93,6 +93,12 @@ class Variable final { return {mutex_, data_}; } + /// Useful for grabbing a reference to an object in a node-based container, + /// e.g. `std::unordered_map`. Values must support concurrent modification. + LockedPtr, Data> SharedMutableLockUnsafe() { + return {mutex_, data_}; + } + LockedPtr, Data> Lock() { return {mutex_, data_}; } LockedPtr, const Data> Lock() const { diff --git a/core/include/userver/utils/statistics/histogram_view.hpp b/core/include/userver/utils/statistics/histogram_view.hpp index 4c5e53e87504..754607a10522 100644 --- a/core/include/userver/utils/statistics/histogram_view.hpp +++ b/core/include/userver/utils/statistics/histogram_view.hpp @@ -26,8 +26,8 @@ struct Access; class HistogramView final { public: // trivially copyable - HistogramView(const HistogramView&) noexcept = default; - HistogramView& operator=(const HistogramView&) noexcept = default; + constexpr HistogramView(const HistogramView&) noexcept = default; + constexpr HistogramView& operator=(const HistogramView&) noexcept = default; /// Returns the number of "normal" (non-"infinity") buckets. std::size_t GetBucketCount() const noexcept; @@ -48,7 +48,9 @@ class HistogramView final { private: friend struct impl::histogram::Access; - explicit HistogramView(const impl::histogram::Bucket* buckets) noexcept; + constexpr explicit HistogramView( + const impl::histogram::Bucket& buckets) noexcept + : buckets_(&buckets) {} const impl::histogram::Bucket* buckets_; }; diff --git a/core/include/userver/utils/statistics/metric_value.hpp b/core/include/userver/utils/statistics/metric_value.hpp index 0d718db25b61..0ee86215728d 100644 --- a/core/include/userver/utils/statistics/metric_value.hpp +++ b/core/include/userver/utils/statistics/metric_value.hpp @@ -21,16 +21,34 @@ class MetricValue final { public: using RawType = std::variant; + /// Creates an unspecified metric value. + constexpr MetricValue() noexcept : value_(std::int64_t{0}) {} + + /// Constructs MetricValue for tests. + /// @{ + constexpr /*implicit*/ MetricValue(std::int64_t value) noexcept + : value_(value) {} + + constexpr /*implicit*/ MetricValue(double value) noexcept : value_(value) {} + + constexpr /*implicit*/ MetricValue(Rate value) noexcept : value_(value) {} + + constexpr /*implicit*/ MetricValue(HistogramView value) noexcept + : value_(value) {} + /// @} + // trivially copyable MetricValue(const MetricValue&) = default; MetricValue& operator=(const MetricValue&) = default; - bool operator==(const MetricValue& other) const noexcept { - return value_ == other.value_; + friend bool operator==(const MetricValue& lhs, + const MetricValue& rhs) noexcept { + return lhs.value_ == rhs.value_; } - bool operator!=(const MetricValue& other) const noexcept { - return value_ != other.value_; + friend bool operator!=(const MetricValue& lhs, + const MetricValue& rhs) noexcept { + return lhs.value_ != rhs.value_; } /// @brief Retrieve the value of an integer metric. @@ -65,8 +83,6 @@ class MetricValue final { } /// @cond - MetricValue() noexcept : value_(std::int64_t{0}) {} - explicit MetricValue(RawType value) noexcept : value_(value) {} /// @endcond diff --git a/core/include/userver/utils/statistics/striped_rate_counter.hpp b/core/include/userver/utils/statistics/striped_rate_counter.hpp index 2b0eb2f8df54..d93e4ba1bf0d 100644 --- a/core/include/userver/utils/statistics/striped_rate_counter.hpp +++ b/core/include/userver/utils/statistics/striped_rate_counter.hpp @@ -90,7 +90,7 @@ class StripedRateCounter final { return val_.Read() + offset_.load(std::memory_order_relaxed); } - concurrent::StripedCounter val_; + USERVER_NAMESPACE::concurrent::StripedCounter val_; std::atomic offset_{0}; }; diff --git a/core/src/utils/statistics/histogram_view.cpp b/core/src/utils/statistics/histogram_view.cpp index 82b4fa0a9a95..0355e71f9dcf 100644 --- a/core/src/utils/statistics/histogram_view.cpp +++ b/core/src/utils/statistics/histogram_view.cpp @@ -16,11 +16,6 @@ static_assert(std::is_trivially_copyable_v && "HistogramView should fit in registers, because it is expected " "to be passed around by value"); -HistogramView::HistogramView(const impl::histogram::Bucket* buckets) noexcept - : buckets_(buckets) { - UASSERT(buckets); -} - std::size_t HistogramView::GetBucketCount() const noexcept { UASSERT(buckets_); return buckets_[0].upper_bound.size; diff --git a/core/src/utils/statistics/impl/histogram_view_utils.hpp b/core/src/utils/statistics/impl/histogram_view_utils.hpp index 99ef2c333fc3..4d41e7b80ed3 100644 --- a/core/src/utils/statistics/impl/histogram_view_utils.hpp +++ b/core/src/utils/statistics/impl/histogram_view_utils.hpp @@ -21,7 +21,8 @@ namespace utils::statistics::impl::histogram { struct Access final { static HistogramView MakeView(const Bucket* buckets) noexcept { - return HistogramView{buckets}; + UASSERT(buckets); + return HistogramView{*buckets}; } template diff --git a/core/utest/include/userver/utils/statistics/testing.hpp b/core/utest/include/userver/utils/statistics/testing.hpp index ccdfe2157858..1757acdfccf7 100644 --- a/core/utest/include/userver/utils/statistics/testing.hpp +++ b/core/utest/include/userver/utils/statistics/testing.hpp @@ -4,6 +4,7 @@ /// @brief Utilities for analyzing emitted metrics in unit tests #include +#include #include #include #include diff --git a/grpc/include/userver/ugrpc/byte_buffer_utils.hpp b/grpc/include/userver/ugrpc/byte_buffer_utils.hpp new file mode 100644 index 000000000000..a346032695fb --- /dev/null +++ b/grpc/include/userver/ugrpc/byte_buffer_utils.hpp @@ -0,0 +1,38 @@ +#pragma once + +/// @file userver/ugrpc/byte_buffer_utils.hpp +/// @brief Helper functions for working with `grpc::ByteBuffer` + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc { + +/// @see @ref SerializeToByteBuffer +inline constexpr std::size_t kDefaultSerializeBlockSize = 4096; + +/// @brief Serialize a Protobuf message to the wire format. +/// @param message the message to serialize +/// @param block_size can be used for performance tuning, too small chunk size +/// results in extra allocations, too large chunk size results in wasting memory +/// @throws std::runtime_error on serialization errors (supposedly only happens +/// on extremely rare allocation failures or proto reflection malfunctioning). +grpc::ByteBuffer SerializeToByteBuffer( + const ::google::protobuf::Message& message, + std::size_t block_size = kDefaultSerializeBlockSize); + +/// @brief Parse a Protobuf message from the wire format. +/// @param buffer the buffer that might be tempered with during deserialization +/// @param message will contain the parsing result on success +/// @returns `true` on success, `false` if @a buffer does not contain a valid +/// message, according to the derived type of @a message +bool ParseFromByteBuffer(grpc::ByteBuffer&& buffer, + ::google::protobuf::Message& message); + +} // namespace ugrpc + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/client_factory.hpp b/grpc/include/userver/ugrpc/client/client_factory.hpp index 63d88ec35639..3f4e113dcc16 100644 --- a/grpc/include/userver/ugrpc/client/client_factory.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory.hpp @@ -50,7 +50,9 @@ struct ClientFactorySettings final { std::size_t channel_count{1}; }; -/// @brief Creates generated gRPC clients. Has a minimal built-in channel cache: +/// @ingroup userver_clients +/// +/// @brief Creates gRPC clients. Has a minimal built-in channel cache: /// as long as a channel to the same endpoint is used somewhere, the same /// channel is given out. class ClientFactory final { @@ -84,17 +86,16 @@ class ClientFactory final { template Client ClientFactory::MakeClient(const std::string& client_name, const std::string& endpoint) { - auto& statistics = client_statistics_storage_.GetServiceStatistics( - Client::GetMetadata(), endpoint); - - Middlewares mws; - mws.reserve(mws_.size()); - for (const auto& mw_factory : mws_) - mws.push_back(mw_factory->GetMiddleware(client_name)); - return Client(impl::ClientParams{ - client_name, std::move(mws), queue_, statistics, - GetChannel(client_name, endpoint), config_source_, testsuite_grpc_}); + client_name, + endpoint, + impl::InstantiateMiddlewares(mws_, client_name), + queue_, + client_statistics_storage_, + GetChannel(client_name, endpoint), + config_source_, + testsuite_grpc_, + }); } } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/generic.hpp b/grpc/include/userver/ugrpc/client/generic.hpp new file mode 100644 index 000000000000..7a1222f236f6 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/generic.hpp @@ -0,0 +1,88 @@ +#pragma once + +/// @file userver/ugrpc/client/generic.hpp +/// @brief @copybrief ugrpc::client::GenericClient + +#include +#include + +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +struct GenericOptions { + /// Client QOS for this call. Note that there is no QOS dynamic config by + /// default, so unless a timeout is specified here, only the deadline + /// propagation mechanism will affect the gRPC deadline. + Qos qos{}; + + /// If non-`nullopt`, metrics are accounted for specified fake call name. + /// If `nullopt` (by default), writes a set of metrics per real call name. + /// If the microservice serves as a proxy and has untrusted clients, it is + /// a good idea to have this option set to non-`nullopt` to avoid + /// the situations where an upstream client can spam various RPCs with + /// non-existent names, which leads to this microservice spamming RPCs + /// with non-existent names, which leads to creating storage for infinite + /// metrics and causes OOM. + std::optional metrics_call_name{}; +}; + +/// @ingroup userver_clients +/// +/// @brief Allows to talk to gRPC services (generic and normal) using dynamic +/// method names. +/// +/// Created using @ref ClientFactory::MakeClient. +/// +/// `call_name` must be in the format `full.path.to.TheService/MethodName`. +/// Note that unlike in base grpc++, there must be no initial `/` character. +/// +/// The API is mainly intended for proxies, where the request-response body is +/// passed unchanged, with settings taken solely from the RPC metadata. +/// In cases where the code needs to operate on the actual messages, +/// serialization of requests and responses is left as an excercise to the user. +/// +/// Middlewares are customizable and are applied as usual, except that no +/// message hooks are called, meaning that there won't be any logs of messages +/// from the default middleware. +/// +/// Metrics are written per-method by default, which causes OOM in some corner +/// cases, for details see @ref GenericOptions::metrics_call_name. +/// +/// ## Example GenericClient usage with known message types +/// +/// @snippet grpc/tests/src/generic_client_test.cpp sample +class GenericClient final { + public: + GenericClient(GenericClient&&) noexcept = default; + GenericClient& operator=(GenericClient&&) noexcept = delete; + + /// Initiate a `single request -> single response` RPC with the given name. + client::UnaryCall UnaryCall( + std::string_view call_name, const grpc::ByteBuffer& request, + std::unique_ptr context = + std::make_unique(), + const GenericOptions& options = {}) const; + + /// @cond + // For internal use only. + explicit GenericClient(impl::ClientParams&&); + /// @endcond + + private: + template + friend impl::ClientData& impl::GetClientData(Client& client); + + impl::ClientData impl_; +}; + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp index cbcefd4d122a..ac8ca92ca621 100644 --- a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp +++ b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -45,27 +46,6 @@ using RawReaderWriter = std::unique_ptr>; /// @} -/// @{ -/// @brief Helper type aliases for stub member function pointers -template -using RawResponseReaderPreparer = RawResponseReader (Stub::*)( - grpc::ClientContext*, const Request&, grpc::CompletionQueue*); - -template -using RawReaderPreparer = RawReader (Stub::*)(grpc::ClientContext*, - const Request&, - grpc::CompletionQueue*); - -template -using RawWriterPreparer = RawWriter (Stub::*)(grpc::ClientContext*, - Response*, - grpc::CompletionQueue*); - -template -using RawReaderWriterPreparer = RawReaderWriter (Stub::*)( - grpc::ClientContext*, grpc::CompletionQueue*); -/// @} - struct RpcConfigValues final { explicit RpcConfigValues(const dynamic_config::Snapshot& config); @@ -156,7 +136,7 @@ class RpcData final { private: std::unique_ptr context_; std::string client_name_; - std::string_view call_name_; + ugrpc::impl::MaybeOwnedString call_name_; bool writes_finished_{false}; bool is_finished_{false}; bool is_deadline_propagated_{false}; diff --git a/grpc/include/userver/ugrpc/client/impl/call_params.hpp b/grpc/include/userver/ugrpc/client/impl/call_params.hpp index 659dc166f70a..5506ec1556ac 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_params.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_params.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -8,8 +9,10 @@ #include #include +#include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -20,37 +23,22 @@ struct CallParams { std::string_view client_name; grpc::CompletionQueue& queue; dynamic_config::Snapshot config; - std::string_view call_name; + ugrpc::impl::MaybeOwnedString call_name; std::unique_ptr context; ugrpc::impl::MethodStatistics& statistics; const Middlewares& mws; }; -CallParams DoCreateCallParams(const ClientData&, std::size_t method_id, - std::unique_ptr); - -template CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id, std::unique_ptr client_context, - const ClientQosConfig& client_qos, - const ugrpc::client::Qos& qos) { - const auto& metadata = client_data.GetMetadata(); - const auto& full_name = metadata.method_full_names[method_id]; - const auto& method_name = - full_name.substr(metadata.service_full_name.size() + 1); - - const auto& config = client_data.GetConfigSnapshot(); - - // User qos goes first - ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); - - // If user qos was empty update timeout from config - ApplyQos(*client_context, config[client_qos][method_name], - client_data.GetTestsuiteControl()); + const dynamic_config::Key& client_qos, + const Qos& qos); - return DoCreateCallParams(client_data, method_id, std::move(client_context)); -} +CallParams CreateGenericCallParams( + const ClientData& client_data, std::string_view call_name, + std::unique_ptr client_context, const Qos& qos, + std::optional metrics_call_name); } // namespace ugrpc::client::impl diff --git a/grpc/include/userver/ugrpc/client/impl/client_data.hpp b/grpc/include/userver/ugrpc/client/impl/client_data.hpp index a1d354b14c66..a0f4b2a89349 100644 --- a/grpc/include/userver/ugrpc/client/impl/client_data.hpp +++ b/grpc/include/userver/ugrpc/client/impl/client_data.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,18 +19,25 @@ USERVER_NAMESPACE_BEGIN +namespace ugrpc::impl { +class StatisticsStorage; +} // namespace ugrpc::impl + namespace ugrpc::client::impl { struct ClientParams final { std::string client_name; + std::string endpoint; Middlewares mws; grpc::CompletionQueue& queue; - ugrpc::impl::ServiceStatistics& statistics_storage; + ugrpc::impl::StatisticsStorage& statistics_storage; impl::ChannelCache::Token channel_token; const dynamic_config::Source config_source; testsuite::GrpcControl& testsuite_grpc; }; +struct GenericClientTag final {}; + /// A helper class for generated gRPC clients class ClientData final { public: @@ -41,14 +49,16 @@ class ClientData final { template ClientData(ClientParams&& params, ugrpc::impl::StaticServiceMetadata metadata, std::in_place_type_t) - : params_(std::move(params)), metadata_(metadata) { - const std::size_t channel_count = GetChannelToken().GetChannelCount(); - stubs_ = utils::GenerateFixedArray(channel_count, [&](std::size_t index) { - return StubPtr( - Service::NewStub(GetChannelToken().GetChannel(index)).release(), - &StubDeleter); - }); - } + : params_(std::move(params)), + metadata_(metadata), + service_statistics_(&GetServiceStatistics()), + stubs_(MakeStubs(params_.channel_token)) {} + + template + ClientData(ClientParams&& params, GenericClientTag, + std::in_place_type_t) + : params_(std::move(params)), + stubs_(MakeStubs(params_.channel_token)) {} ClientData(ClientData&&) noexcept = default; ClientData& operator=(ClientData&&) = delete; @@ -68,9 +78,10 @@ class ClientData final { return params_.config_source.GetSnapshot(); } - ugrpc::impl::MethodStatistics& GetStatistics(std::size_t method_id) const { - return params_.statistics_storage.GetMethodStatistics(method_id); - } + ugrpc::impl::MethodStatistics& GetStatistics(std::size_t method_id) const; + + ugrpc::impl::MethodStatistics& GetGenericStatistics( + std::string_view call_name) const; ChannelCache::Token& GetChannelToken() { return params_.channel_token; } @@ -78,9 +89,7 @@ class ClientData final { const Middlewares& GetMiddlewares() const { return params_.mws; } - const ugrpc::impl::StaticServiceMetadata& GetMetadata() const { - return metadata_; - } + const ugrpc::impl::StaticServiceMetadata& GetMetadata() const; const testsuite::GrpcControl& GetTestsuiteControl() const { return params_.testsuite_grpc; @@ -95,8 +104,22 @@ class ClientData final { delete static_cast*>(ptr); } + template + static utils::FixedArray MakeStubs( + impl::ChannelCache::Token& channel_token) { + const std::size_t channel_count = channel_token.GetChannelCount(); + return utils::GenerateFixedArray(channel_count, [&](std::size_t index) { + return StubPtr( + Service::NewStub(channel_token.GetChannel(index)).release(), + &StubDeleter); + }); + } + + ugrpc::impl::ServiceStatistics& GetServiceStatistics(); + ClientParams params_; - ugrpc::impl::StaticServiceMetadata metadata_; + std::optional metadata_{std::nullopt}; + ugrpc::impl::ServiceStatistics* service_statistics_{nullptr}; utils::FixedArray stubs_; }; diff --git a/grpc/include/userver/ugrpc/client/middlewares/base.hpp b/grpc/include/userver/ugrpc/client/middlewares/base.hpp index f19fdc8d2d45..90cb469f4853 100644 --- a/grpc/include/userver/ugrpc/client/middlewares/base.hpp +++ b/grpc/include/userver/ugrpc/client/middlewares/base.hpp @@ -94,6 +94,13 @@ class MiddlewareComponentBase : public components::ComponentBase { GetMiddlewareFactory() = 0; }; +namespace impl { + +Middlewares InstantiateMiddlewares(const MiddlewareFactories& factories, + const std::string& client_name); + +} // namespace impl + } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp b/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp index 66cadb8678b7..6f29417fb25b 100644 --- a/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp +++ b/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp @@ -4,12 +4,15 @@ /// @brief @copybrief ugrpc::client::middlewares::log::Component #include +#include USERVER_NAMESPACE_BEGIN /// Client logging middleware namespace ugrpc::client::middlewares::log { +struct Settings; + // clang-format off /// @ingroup userver_components @@ -33,13 +36,14 @@ class Component final : public MiddlewareComponentBase { Component(const components::ComponentConfig& config, const components::ComponentContext& context); + ~Component() override; + std::shared_ptr GetMiddlewareFactory() override; static yaml_config::Schema GetStaticConfigSchema(); private: - std::size_t max_size_; - logging::Level log_level_; + const utils::Box settings_; }; } // namespace ugrpc::client::middlewares::log diff --git a/grpc/include/userver/ugrpc/client/rpc.hpp b/grpc/include/userver/ugrpc/client/rpc.hpp index c63f675f3ff4..a9262d62f703 100644 --- a/grpc/include/userver/ugrpc/client/rpc.hpp +++ b/grpc/include/userver/ugrpc/client/rpc.hpp @@ -180,11 +180,9 @@ class [[nodiscard]] UnaryCall final : public CallAnyBase { /// @cond // For internal use only - template - UnaryCall( - impl::CallParams&& params, Stub& stub, - impl::RawResponseReaderPreparer prepare_func, - const Request& req); + template + UnaryCall(impl::CallParams&& params, PrepareFunc prepare_func, + const Request& req); /// @endcond UnaryCall(UnaryCall&&) noexcept = default; @@ -223,9 +221,8 @@ class [[nodiscard]] InputStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncReader; - template - InputStream(impl::CallParams&& params, Stub& stub, - impl::RawReaderPreparer prepare_func, + template + InputStream(impl::CallParams&& params, PrepareFunc prepare_func, const Request& req); /// @endcond @@ -293,9 +290,8 @@ class [[nodiscard]] OutputStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncWriter; - template - OutputStream(impl::CallParams&& params, Stub& stub, - impl::RawWriterPreparer prepare_func); + template + OutputStream(impl::CallParams&& params, PrepareFunc prepare_func); /// @endcond OutputStream(OutputStream&&) noexcept = default; @@ -399,10 +395,8 @@ class [[nodiscard]] BidirectionalStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncReaderWriter; - template - BidirectionalStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderWriterPreparer prepare_func); + template + BidirectionalStream(impl::CallParams&& params, PrepareFunc prepare_func); /// @endcond BidirectionalStream(BidirectionalStream&&) noexcept = default; @@ -477,20 +471,22 @@ bool StreamReadFuture::IsReady() const noexcept { } template -template -UnaryCall::UnaryCall( - impl::CallParams&& params, Stub& stub, - impl::RawResponseReaderPreparer prepare_func, - const Request& req) +template +UnaryCall::UnaryCall(impl::CallParams&& params, + PrepareFunc prepare_func, const Request& req) : CallAnyBase(std::move(params)) { + const ::google::protobuf::Message* req_message = nullptr; + if constexpr (std::is_base_of_v<::google::protobuf::Message, Request>) { + req_message = &req; + } impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - reader_ = (stub.*prepare_func)(&GetData().GetContext(), req, - &GetData().GetQueue()); + reader_ = + prepare_func(&GetData().GetContext(), req, &GetData().GetQueue()); reader_->StartCall(); }, - &req); + req_message); GetData().SetWritesFinished(); } @@ -514,20 +510,22 @@ UnaryFuture UnaryCall::FinishAsync(Response& response) { } template -template -InputStream::InputStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderPreparer prepare_func, - const Request& req) +template +InputStream::InputStream(impl::CallParams&& params, + PrepareFunc prepare_func, const Request& req) : CallAnyBase(std::move(params)) { + const ::google::protobuf::Message* req_message = nullptr; + if constexpr (std::is_base_of_v<::google::protobuf::Message, Request>) { + req_message = &req; + } impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - stream_ = (stub.*prepare_func)(&GetData().GetContext(), req, - &GetData().GetQueue()); + stream_ = + prepare_func(&GetData().GetContext(), req, &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, - &req); + req_message); GetData().SetWritesFinished(); } @@ -544,19 +542,17 @@ bool InputStream::Read(Response& response) { } template -template -OutputStream::OutputStream( - impl::CallParams&& params, Stub& stub, - impl::RawWriterPreparer prepare_func) +template +OutputStream::OutputStream(impl::CallParams&& params, + PrepareFunc prepare_func) : CallAnyBase(std::move(params)), final_response_(std::make_unique()) { impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { // 'final_response_' will be filled upon successful 'Finish' async call - stream_ = - (stub.*prepare_func)(&GetData().GetContext(), final_response_.get(), - &GetData().GetQueue()); + stream_ = prepare_func(&GetData().GetContext(), final_response_.get(), + &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, nullptr); @@ -596,16 +592,14 @@ Response OutputStream::Finish() { } template -template +template BidirectionalStream::BidirectionalStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderWriterPreparer prepare_func) + impl::CallParams&& params, PrepareFunc prepare_func) : CallAnyBase(std::move(params)) { impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - stream_ = (stub.*prepare_func)(&GetData().GetContext(), - &GetData().GetQueue()); + stream_ = prepare_func(&GetData().GetContext(), &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, nullptr); diff --git a/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp b/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp new file mode 100644 index 000000000000..abca68678944 --- /dev/null +++ b/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::impl { + +class MaybeOwnedString final { + public: + struct Ref {}; + + MaybeOwnedString() = default; + + explicit MaybeOwnedString(std::string&& string) + : storage_(std::move(string)), view_(storage_) {} + + MaybeOwnedString(Ref, std::string_view string) : view_(string) {} + + MaybeOwnedString(MaybeOwnedString&& other) noexcept { + *this = std::move(other); + } + + MaybeOwnedString& operator=(MaybeOwnedString&& other) noexcept { + if (this == &other) return *this; + if (other.view_.data() == other.storage_.data()) { + storage_ = std::move(other.storage_); + view_ = storage_; + } else { + storage_.clear(); + view_ = other.view_; + } + return *this; + } + + std::string_view Get() const noexcept { return view_; } + + private: + std::string storage_; + std::string_view view_; +}; + +} // namespace ugrpc::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/impl/statistics.hpp b/grpc/include/userver/ugrpc/impl/statistics.hpp index ae61e24edda4..9eeeaabc9497 100644 --- a/grpc/include/userver/ugrpc/impl/statistics.hpp +++ b/grpc/include/userver/ugrpc/impl/statistics.hpp @@ -18,6 +18,10 @@ USERVER_NAMESPACE_BEGIN +namespace utils::statistics { +class StripedRateCounter; +} // namespace utils::statistics + namespace ugrpc::impl { enum class StatisticsDomain { kClient, kServer }; @@ -26,7 +30,9 @@ std::string_view ToString(StatisticsDomain); class MethodStatistics final { public: - explicit MethodStatistics(StatisticsDomain domain); + explicit MethodStatistics( + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started); void AccountStarted() noexcept; @@ -57,6 +63,7 @@ class MethodStatistics final { private: using Percentile = utils::statistics::Percentile<2000, std::uint32_t, 256, 100>; + using Timings = utils::statistics::RecentPeriod; using RateCounter = utils::statistics::RateCounter; // StatusCode enum cases have consecutive underlying values, starting from 0. // UNAUTHENTICATED currently has the largest value. @@ -64,9 +71,11 @@ class MethodStatistics final { static_cast(grpc::StatusCode::UNAUTHENTICATED) + 1; const StatisticsDomain domain_; + utils::statistics::StripedRateCounter& global_started_; + RateCounter started_{0}; std::array status_codes_{}; - utils::statistics::RecentPeriod timings_; + Timings timings_; RateCounter network_errors_{0}; RateCounter internal_errors_{0}; RateCounter cancelled_{0}; @@ -78,7 +87,8 @@ class MethodStatistics final { class ServiceStatistics final { public: ServiceStatistics(const StaticServiceMetadata& metadata, - StatisticsDomain domain); + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started); ~ServiceStatistics(); diff --git a/grpc/include/userver/ugrpc/impl/statistics_storage.hpp b/grpc/include/userver/ugrpc/impl/statistics_storage.hpp index 5d9232f62646..5f24e96c3ab4 100644 --- a/grpc/include/userver/ugrpc/impl/statistics_storage.hpp +++ b/grpc/include/userver/ugrpc/impl/statistics_storage.hpp @@ -5,8 +5,11 @@ #include #include +#include #include +#include #include +#include #include @@ -14,8 +17,7 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { -/// Clients are created on-the-fly, so we must use a separate stable container -/// for storing their statistics. +/// Allows to create ServiceStatistics and generic MethodStatistics on the fly. class StatisticsStorage final { public: explicit StatisticsStorage(utils::statistics::Storage& statistics_storage, @@ -23,16 +25,14 @@ class StatisticsStorage final { StatisticsStorage(const StatisticsStorage&) = delete; StatisticsStorage& operator=(const StatisticsStorage&) = delete; - ~StatisticsStorage(); - ugrpc::impl::ServiceStatistics& GetServiceStatistics( - const ugrpc::impl::StaticServiceMetadata& metadata, - std::optional endpoint); + ServiceStatistics& GetServiceStatistics(const StaticServiceMetadata& metadata, + std::optional endpoint); + + MethodStatistics& GetGenericStatistics( + std::string_view call_name, std::optional endpoint); - // Can only be called on StatisticsStorage for gRPC services (not clients). - // Can only be called strictly after all the components are loaded. - // gRPC services must not be [un]registered during GetStartedRequests(). std::uint64_t GetStartedRequests() const; private: @@ -40,27 +40,59 @@ class StatisticsStorage final { using ServiceId = const char*; struct ServiceKey { - ServiceId service_id; + ServiceId service_id{}; std::optional endpoint; }; - void ExtendStatistics(utils::statistics::Writer& writer); + struct GenericKey { + std::string call_name; + std::optional endpoint; + }; - struct ServiceKeyComparer final { + struct GenericKeyView { + GenericKey Dereference() const; + + std::string_view call_name; + std::optional endpoint; + }; + + struct ServiceKeyComparer { bool operator()(ServiceKey lhs, ServiceKey rhs) const; }; - struct ServiceKeyHasher final { - std::size_t operator()(const ServiceKey& key) const; + struct ServiceKeyHasher { + std::size_t operator()(const ServiceKey& key) const noexcept; }; - const StatisticsDomain domain_; + struct GenericKeyComparer { + using is_transparent [[maybe_unused]] = void; - std::unordered_map - service_statistics_; - engine::SharedMutex mutex_; + bool operator()(const GenericKey& lhs, const GenericKey& rhs) const; + bool operator()(const GenericKeyView& lhs, const GenericKey& rhs) const; + bool operator()(const GenericKey& lhs, const GenericKeyView& rhs) const; + }; + + struct GenericKeyHasher { + using is_transparent [[maybe_unused]] = void; + + std::size_t operator()(const GenericKey& key) const noexcept; + std::size_t operator()(const GenericKeyView& key) const noexcept; + }; + void ExtendStatistics(utils::statistics::Writer& writer); + + const StatisticsDomain domain_; + utils::statistics::StripedRateCounter global_started_; + concurrent::Variable, + engine::SharedMutex> + service_statistics_; + concurrent::Variable< + utils::impl::TransparentMap, + engine::SharedMutex> + generic_statistics_; + // statistics_holder_ must be the last field. utils::statistics::Entry statistics_holder_; }; diff --git a/grpc/src/ugrpc/byte_buffer_utils.cpp b/grpc/src/ugrpc/byte_buffer_utils.cpp new file mode 100644 index 000000000000..ab2cf1534aab --- /dev/null +++ b/grpc/src/ugrpc/byte_buffer_utils.cpp @@ -0,0 +1,40 @@ +#include + +#include +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc { + +grpc::ByteBuffer SerializeToByteBuffer( + const ::google::protobuf::Message& message, std::size_t block_size) { + grpc::ByteBuffer buffer; + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject) + grpc::ProtoBufferWriter writer{ + &buffer, + /*block_size*/ utils::numeric_cast(block_size), + /*total_size*/ utils::numeric_cast(message.ByteSizeLong()), + }; + const bool success = message.SerializeToZeroCopyStream(&writer); + if (!success) { + throw std::runtime_error( + fmt::format("Failed to serialize Protobuf message of type {}", + message.GetTypeName())); + } + return buffer; +} + +bool ParseFromByteBuffer(grpc::ByteBuffer&& buffer, + ::google::protobuf::Message& message) { + grpc::ProtoBufferReader reader{&buffer}; + return message.ParseFromZeroCopyStream(&reader); +} + +} // namespace ugrpc + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/generic.cpp b/grpc/src/ugrpc/client/generic.cpp new file mode 100644 index 000000000000..ef652d9eb6b9 --- /dev/null +++ b/grpc/src/ugrpc/client/generic.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +namespace { + +struct GenericStubService final { + using Stub = grpc::GenericStub; + + static std::unique_ptr NewStub(std::shared_ptr channel) { + return std::make_unique(channel); + } +}; + +} // namespace + +GenericClient::GenericClient(impl::ClientParams&& client_params) + : impl_(std::move(client_params), impl::GenericClientTag{}, + std::in_place_type) {} + +client::UnaryCall GenericClient::UnaryCall( + std::string_view call_name, const grpc::ByteBuffer& request, + std::unique_ptr context, + const GenericOptions& generic_options) const { + auto& stub = impl_.NextStub(); + auto grpcpp_call_name = utils::StrCat("/", call_name); + return { + impl::CreateGenericCallParams(impl_, call_name, std::move(context), + generic_options.qos, + generic_options.metrics_call_name), + [&stub, &grpcpp_call_name](grpc::ClientContext* context, + const grpc::ByteBuffer& request, + grpc::CompletionQueue* cq) { + return stub.PrepareUnaryCall(context, grpcpp_call_name, request, cq); + }, + request, + }; +} + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp index 888afe83e294..06ceab588df3 100644 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_methods.cpp @@ -88,15 +88,15 @@ void FutureImpl::ClearData() noexcept { data_ = nullptr; } RpcData::RpcData(impl::CallParams&& params) : context_(std::move(params.context)), - client_name_(params.call_name), - call_name_(params.call_name), + client_name_(params.client_name), + call_name_(std::move(params.call_name)), stats_scope_(params.statistics), queue_(params.queue), config_values_(params.config), mws_(params.mws) { UASSERT(context_); UASSERT(!client_name_.empty()); - SetupSpan(span_, *context_, call_name_); + SetupSpan(span_, *context_, call_name_.Get()); } RpcData::~RpcData() { @@ -136,7 +136,7 @@ const Middlewares& RpcData::GetMiddlewares() const noexcept { std::string_view RpcData::GetCallName() const noexcept { UASSERT(context_); - return call_name_; + return call_name_.Get(); } std::string_view RpcData::GetClientName() const noexcept { @@ -282,7 +282,8 @@ void ProcessFinishResult(RpcData& data, std::move(parsed_gstatus.gstatus_string)); } } else { - data.GetStatsScope().Flush(); + data.GetSpan().AddTag("grpc_code", + std::string{ugrpc::ToString(grpc::StatusCode::OK)}); data.ResetSpan(); } } diff --git a/grpc/src/ugrpc/client/impl/call_params.cpp b/grpc/src/ugrpc/client/impl/call_params.cpp index 82879f106258..561498106847 100644 --- a/grpc/src/ugrpc/client/impl/call_params.cpp +++ b/grpc/src/ugrpc/client/impl/call_params.cpp @@ -1,19 +1,77 @@ #include +#include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -CallParams DoCreateCallParams(const ClientData& client_data, - std::size_t method_id, - std::unique_ptr context) { - return CallParams{client_data.GetClientName(), - client_data.GetQueue(), - client_data.GetConfigSnapshot(), - client_data.GetMetadata().method_full_names[method_id], - std::move(context), - client_data.GetStatistics(method_id), - client_data.GetMiddlewares()}; +namespace { + +void CheckValidCallName(std::string_view call_name) { + UASSERT_MSG(!call_name.empty(), "generic call_name must NOT be empty"); + UASSERT_MSG(call_name[0] != '/', + utils::StrCat("generic call_name must NOT start with /, given: ", + call_name)); + UASSERT_MSG( + call_name.find('/') != std::string_view::npos, + utils::StrCat("generic call_name must contain /, given: ", call_name)); +} + +} // namespace + +CallParams CreateCallParams(const ClientData& client_data, + std::size_t method_id, + std::unique_ptr client_context, + const dynamic_config::Key& client_qos, + const Qos& qos) { + const auto& metadata = client_data.GetMetadata(); + const auto call_name = metadata.method_full_names[method_id]; + const auto method_name = + call_name.substr(metadata.service_full_name.size() + 1); + + const auto& config = client_data.GetConfigSnapshot(); + + // User qos goes first + ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); + + // If user qos was empty update timeout from config + ApplyQos(*client_context, config[client_qos][method_name], + client_data.GetTestsuiteControl()); + + return CallParams{ + client_data.GetClientName(), // + client_data.GetQueue(), + client_data.GetConfigSnapshot(), + {ugrpc::impl::MaybeOwnedString::Ref{}, call_name}, + std::move(client_context), + client_data.GetStatistics(method_id), + client_data.GetMiddlewares(), + }; +} + +CallParams CreateGenericCallParams( + const ClientData& client_data, std::string_view call_name, + std::unique_ptr client_context, const Qos& qos, + std::optional metrics_call_name) { + CheckValidCallName(call_name); + if (metrics_call_name) { + CheckValidCallName(*metrics_call_name); + } + + // User qos goes first + ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); + + return CallParams{ + client_data.GetClientName(), // + client_data.GetQueue(), + client_data.GetConfigSnapshot(), + ugrpc::impl::MaybeOwnedString{std::string{call_name}}, + std::move(client_context), + client_data.GetGenericStatistics(metrics_call_name.value_or(call_name)), + client_data.GetMiddlewares(), + }; } } // namespace ugrpc::client::impl diff --git a/grpc/src/ugrpc/client/impl/client_data.cpp b/grpc/src/ugrpc/client/impl/client_data.cpp new file mode 100644 index 000000000000..90ffb1ce5d61 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/client_data.cpp @@ -0,0 +1,34 @@ +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +ugrpc::impl::MethodStatistics& ClientData::GetStatistics( + std::size_t method_id) const { + UASSERT(service_statistics_); + return service_statistics_->GetMethodStatistics(method_id); +} + +ugrpc::impl::MethodStatistics& ClientData::GetGenericStatistics( + std::string_view call_name) const { + return params_.statistics_storage.GetGenericStatistics(call_name, + params_.endpoint); +} + +const ugrpc::impl::StaticServiceMetadata& ClientData::GetMetadata() const { + UASSERT(metadata_); + return *metadata_; +} + +ugrpc::impl::ServiceStatistics& ClientData::GetServiceStatistics() { + return params_.statistics_storage.GetServiceStatistics(GetMetadata(), + params_.endpoint); +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/middlewares/base.cpp b/grpc/src/ugrpc/client/middlewares/base.cpp index f641abe222ed..4d0762296ce1 100644 --- a/grpc/src/ugrpc/client/middlewares/base.cpp +++ b/grpc/src/ugrpc/client/middlewares/base.cpp @@ -40,6 +40,20 @@ const ::google::protobuf::Message* MiddlewareCallContext::GetInitialRequest() { MiddlewareFactoryBase::~MiddlewareFactoryBase() = default; +namespace impl { + +Middlewares InstantiateMiddlewares(const MiddlewareFactories& factories, + const std::string& client_name) { + Middlewares mws; + mws.reserve(factories.size()); + for (const auto& mw_factory : factories) { + mws.push_back(mw_factory->GetMiddleware(client_name)); + } + return mws; +} + +} // namespace impl + } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/middlewares/log/component.cpp b/grpc/src/ugrpc/client/middlewares/log/component.cpp index 4943cf140200..d8db078f3a3e 100644 --- a/grpc/src/ugrpc/client/middlewares/log/component.cpp +++ b/grpc/src/ugrpc/client/middlewares/log/component.cpp @@ -9,18 +9,27 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client::middlewares::log { +Settings Parse(const yaml_config::YamlConfig& config, + formats::parse::To) { + Settings settings; + settings.max_msg_size = + config["msg-size-log-limit"].As(settings.max_msg_size); + settings.log_level = + config["log-level"].As(settings.log_level); + return settings; +} + Component::Component(const components::ComponentConfig& config, const components::ComponentContext& context) : MiddlewareComponentBase(config, context), - max_size_(config["msg-size-log-limit"].As(512)) { - log_level_ = config["log-level"].As(logging::Level::kDebug); -} + settings_(config.As()) {} std::shared_ptr Component::GetMiddlewareFactory() { - return std::make_shared( - Middleware::Settings{max_size_, log_level_}); + return std::make_shared(*settings_); } +Component::~Component() = default; + yaml_config::Schema Component::GetStaticConfigSchema() { return yaml_config::MergeSchemas(R"( type: object diff --git a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp index b718d115b1f9..c0dbe3667e57 100644 --- a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp @@ -20,7 +20,7 @@ void Middleware::Handle(MiddlewareCallContext& context) const { context.Next(); } -MiddlewareFactory::MiddlewareFactory(const Middleware::Settings& settings) +MiddlewareFactory::MiddlewareFactory(const Settings& settings) : settings_(settings) {} std::shared_ptr MiddlewareFactory::GetMiddleware( diff --git a/grpc/src/ugrpc/client/middlewares/log/middleware.hpp b/grpc/src/ugrpc/client/middlewares/log/middleware.hpp index f14629a5c2dd..5b3b93a58d90 100644 --- a/grpc/src/ugrpc/client/middlewares/log/middleware.hpp +++ b/grpc/src/ugrpc/client/middlewares/log/middleware.hpp @@ -1,19 +1,24 @@ #pragma once +#include + #include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::middlewares::log { +struct Settings { + /// Max gRPC message size, the rest will be truncated + std::size_t max_msg_size{512}; + + /// gRPC message logging level + logging::Level log_level{logging::Level::kDebug}; +}; + /// @brief middleware for RPC handler logging settings class Middleware final : public MiddlewareBase { public: - struct Settings { - size_t max_msg_size; ///< Max gRPC message size, the rest will be truncated - logging::Level log_level; ///< gRPC message logging level - }; - explicit Middleware(const Settings& settings); void Handle(MiddlewareCallContext& context) const override; @@ -25,13 +30,13 @@ class Middleware final : public MiddlewareBase { /// @cond class MiddlewareFactory final : public MiddlewareFactoryBase { public: - explicit MiddlewareFactory(const Middleware::Settings& settings); + explicit MiddlewareFactory(const Settings& settings); std::shared_ptr GetMiddleware( std::string_view client_name) const override; private: - Middleware::Settings settings_; + Settings settings_; }; /// @endcond diff --git a/grpc/src/ugrpc/impl/statistics.cpp b/grpc/src/ugrpc/impl/statistics.cpp index c9a81993c365..843e18921b70 100644 --- a/grpc/src/ugrpc/impl/statistics.cpp +++ b/grpc/src/ugrpc/impl/statistics.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -47,9 +48,15 @@ void DumpMetric(utils::statistics::Writer& writer, } // namespace -MethodStatistics::MethodStatistics(StatisticsDomain domain) : domain_(domain) {} +MethodStatistics::MethodStatistics( + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started) + : domain_(domain), global_started_(global_started) {} -void MethodStatistics::AccountStarted() noexcept { ++started_; } +void MethodStatistics::AccountStarted() noexcept { + ++started_; + ++global_started_; +} void MethodStatistics::AccountStatus(grpc::StatusCode code) noexcept { if (static_cast(code) < kCodesCount) { @@ -158,10 +165,12 @@ std::uint64_t MethodStatistics::GetStarted() const noexcept { ServiceStatistics::~ServiceStatistics() = default; -ServiceStatistics::ServiceStatistics(const StaticServiceMetadata& metadata, - StatisticsDomain domain) +ServiceStatistics::ServiceStatistics( + const StaticServiceMetadata& metadata, StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started) : metadata_(metadata), - method_statistics_(metadata.method_full_names.size(), domain) {} + method_statistics_(metadata.method_full_names.size(), domain, + global_started) {} MethodStatistics& ServiceStatistics::GetMethodStatistics( std::size_t method_id) { diff --git a/grpc/src/ugrpc/impl/statistics_storage.cpp b/grpc/src/ugrpc/impl/statistics_storage.cpp index f46080f70bcb..897f6e1deea9 100644 --- a/grpc/src/ugrpc/impl/statistics_storage.cpp +++ b/grpc/src/ugrpc/impl/statistics_storage.cpp @@ -1,6 +1,9 @@ #include +#include + #include +#include #include #include @@ -30,17 +33,39 @@ StatisticsStorage::StatisticsStorage( StatisticsStorage::~StatisticsStorage() { statistics_holder_.Unregister(); } -ugrpc::impl::ServiceStatistics& StatisticsStorage::GetServiceStatistics( - const ugrpc::impl::StaticServiceMetadata& metadata, +ServiceStatistics& StatisticsStorage::GetServiceStatistics( + const StaticServiceMetadata& metadata, std::optional endpoint) { // We exploit the fact that 'service_full_name' always points to the same // static string for a given service. const ServiceId service_id = metadata.service_full_name.data(); - const auto service_key = ServiceKey{service_id, endpoint}; + ServiceKey service_key{service_id, std::move(endpoint)}; + + { + auto service_statistics = service_statistics_.SharedMutableLockUnsafe(); + if (auto* stats = utils::FindOrNullptr(*service_statistics, service_key)) { + return *stats; + } + } + + // All the other clients are blocked while we instantiate stats for a new + // service. This is OK, because it will only happen a finite number of times + // during startup. + auto service_statistics = service_statistics_.Lock(); + + const auto [iter, is_new] = service_statistics->try_emplace( + std::move(service_key), metadata, domain_, global_started_); + return iter->second; +} + +MethodStatistics& StatisticsStorage::GetGenericStatistics( + std::string_view call_name, std::optional endpoint) { + const GenericKeyView generic_key{call_name, endpoint}; { - const std::shared_lock lock(mutex_); - if (auto* stats = utils::FindOrNullptr(service_statistics_, service_key)) { + auto generic_statistics = generic_statistics_.SharedMutableLockUnsafe(); + if (auto* stats = utils::impl::FindTransparentOrNullptr(*generic_statistics, + generic_key)) { return *stats; } } @@ -48,18 +73,18 @@ ugrpc::impl::ServiceStatistics& StatisticsStorage::GetServiceStatistics( // All the other clients are blocked while we instantiate stats for a new // service. This is OK, because it will only happen a finite number of times // during startup. - const std::lock_guard lock(mutex_); + auto generic_statistics = generic_statistics_.Lock(); - const auto [iter, is_new] = service_statistics_.try_emplace( - std::move(service_key), metadata, domain_); + const auto [iter, is_new] = generic_statistics->try_emplace( + generic_key.Dereference(), domain_, global_started_); return iter->second; } void StatisticsStorage::ExtendStatistics(utils::statistics::Writer& writer) { - const std::shared_lock lock(mutex_); + auto by_destination = writer["by-destination"]; { - auto by_destination = writer["by-destination"]; - for (const auto& [key, service_stats] : service_statistics_) { + auto service_statistics = service_statistics_.SharedLock(); + for (const auto& [key, service_stats] : *service_statistics) { if (key.endpoint) { by_destination.ValueWithLabels(std::move(service_stats), {"endpoint", *key.endpoint}); @@ -68,17 +93,46 @@ void StatisticsStorage::ExtendStatistics(utils::statistics::Writer& writer) { } } } + { + auto generic_statistics = generic_statistics_.SharedLock(); + for (const auto& [key, service_stats] : *generic_statistics) { + const std::string_view call_name = key.call_name; + + const auto slash_pos = call_name.find('/'); + if (slash_pos == std::string_view::npos || slash_pos == 0) { + UASSERT(false); + continue; + } + + const auto service_name = call_name.substr(0, slash_pos); + const auto method_name = call_name.substr(slash_pos + 1); + + if (key.endpoint) { + by_destination.ValueWithLabels(std::move(service_stats), + {{"grpc_service", service_name}, + {"grpc_method", method_name}, + {"grpc_destination", key.call_name}, + {"endpoint", *key.endpoint}}); + } else { + by_destination.ValueWithLabels(std::move(service_stats), + {{"grpc_service", service_name}, + {"grpc_method", method_name}, + {"grpc_destination", key.call_name}}); + } + } + } } std::uint64_t StatisticsStorage::GetStartedRequests() const { - std::uint64_t result{0}; - // mutex_ is not locked as we might be inside a common thread w/o coroutine - // environment. service_statistics_ is not changed as all gRPC services (not - // clients!) are already registered. - for (const auto& [name, stats] : service_statistics_) { - result += stats.GetStartedRequests(); - } - return result; + return global_started_.Load().value; +} + +StatisticsStorage::GenericKey // +StatisticsStorage::GenericKeyView::Dereference() const { + return GenericKey{ + std::string{call_name}, + endpoint ? std::make_optional(std::string{*endpoint}) : std::nullopt, + }; } bool StatisticsStorage::ServiceKeyComparer::operator()(ServiceKey lhs, @@ -87,9 +141,33 @@ bool StatisticsStorage::ServiceKeyComparer::operator()(ServiceKey lhs, } std::size_t StatisticsStorage::ServiceKeyHasher::operator()( - const ServiceKey& key) const { - return std::hash{}(key.service_id) ^ - std::hash{}(key.endpoint); + const ServiceKey& key) const noexcept { + return boost::hash_value(std::tie(key.service_id, key.endpoint)); +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKey& lhs, const GenericKey& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKeyView& lhs, const GenericKey& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKey& lhs, const GenericKeyView& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +std::size_t StatisticsStorage::GenericKeyHasher::operator()( + const GenericKey& key) const noexcept { + return boost::hash_value(std::tie(key.call_name, key.endpoint)); +} + +std::size_t StatisticsStorage::GenericKeyHasher::operator()( + const GenericKeyView& key) const noexcept { + return boost::hash_value(std::tie(key.call_name, key.endpoint)); } } // namespace ugrpc::impl diff --git a/grpc/tests/src/base_test.cpp b/grpc/tests/src/base_test.cpp index fb11e32a05f8..96ec8a531eca 100644 --- a/grpc/tests/src/base_test.cpp +++ b/grpc/tests/src/base_test.cpp @@ -132,6 +132,7 @@ UTEST_F(GrpcClientTest, UnaryRPC) { out.set_name("userver"); auto call_for_move = client.SayHello(out, PrepareClientContext()); auto call = std::move(call_for_move); // test move operation + EXPECT_EQ(call.GetCallName(), "sample.ugrpc.UnitTestService/SayHello"); sample::ugrpc::GreetingResponse in; UEXPECT_NO_THROW(in = call.Finish()); diff --git a/grpc/tests/src/generic_client_test.cpp b/grpc/tests/src/generic_client_test.cpp new file mode 100644 index 000000000000..e58c939649c2 --- /dev/null +++ b/grpc/tests/src/generic_client_test.cpp @@ -0,0 +1,125 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +const grpc::string kSayHelloCallName = "sample.ugrpc.UnitTestService/SayHello"; + +class UnitTestService final : public sample::ugrpc::UnitTestServiceBase { + public: + void SayHello(SayHelloCall& call, + sample::ugrpc::GreetingRequest&& request) override { + sample::ugrpc::GreetingResponse response; + response.set_name("Hello " + request.name()); + call.Finish(response); + } +}; + +using GenericClientTest = ugrpc::tests::ServiceFixture; + +sample::ugrpc::GreetingResponse PerformGenericUnaryCall( + ugrpc::tests::ServiceBase& test_service) { + auto& client_factory = test_service; + + /// [sample] + const auto client = client_factory.MakeClient(); + + const grpc::string call_name = "sample.ugrpc.UnitTestService/SayHello"; + + sample::ugrpc::GreetingRequest request; + request.set_name("generic"); + + auto rpc = client.UnaryCall(call_name, ugrpc::SerializeToByteBuffer(request)); + + auto response_bytes = rpc.Finish(); + sample::ugrpc::GreetingResponse response; + if (!ugrpc::ParseFromByteBuffer(std::move(response_bytes), response)) { + throw ugrpc::client::RpcError(rpc.GetCallName(), + "Failed to parse response"); + } + + return response; + /// [sample] +} + +} // namespace + +UTEST_F(GenericClientTest, UnaryCall) { + const auto response = PerformGenericUnaryCall(*this); + EXPECT_EQ(response.name(), "Hello generic"); +} + +UTEST_F(GenericClientTest, Metrics) { + PerformGenericUnaryCall(*this); + + const auto stats = GetStatistics( + "grpc.client.by-destination", + {{"grpc_method", "SayHello"}, + {"grpc_service", "sample.ugrpc.UnitTestService"}, + {"grpc_destination", "sample.ugrpc.UnitTestService/SayHello"}}); + UEXPECT_NO_THROW( + EXPECT_EQ(stats.SingleMetric("status", {{"grpc_code", "OK"}}), + utils::statistics::Rate{1}) + << testing::PrintToString(stats)) + << testing::PrintToString(stats); +} + +UTEST_F(GenericClientTest, MetricsCustomCallName) { + const auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("generic"); + + ugrpc::client::GenericOptions options; + options.metrics_call_name = "GenericService/GenericMethod"; + + auto rpc = + client.UnaryCall(kSayHelloCallName, ugrpc::SerializeToByteBuffer(request), + std::make_unique(), options); + EXPECT_EQ(rpc.GetCallName(), kSayHelloCallName); + rpc.Finish(); + + const auto stats = + GetStatistics("grpc.client.by-destination", + {{"grpc_method", "GenericMethod"}, + {"grpc_service", "GenericService"}, + {"grpc_destination", "GenericService/GenericMethod"}}); + UEXPECT_NO_THROW( + EXPECT_EQ(stats.SingleMetric("status", {{"grpc_code", "OK"}}), + utils::statistics::Rate{1}) + << testing::PrintToString(stats)) + << testing::PrintToString(stats); +} + +namespace { + +using GenericClientLoggingTest = + utest::LogCaptureFixture>; + +} // namespace + +UTEST_F(GenericClientLoggingTest, Logs) { + PerformGenericUnaryCall(*this); + + const auto span_log = GetSingleLog( + GetLogCapture().Filter("", {{{std::string_view("stopwatch_name"), + std::string_view("external_grpc")}}})); + EXPECT_EQ(span_log.GetTagOptional("stopwatch_name"), + "external_grpc/sample.ugrpc.UnitTestService/SayHello") + << span_log; + EXPECT_EQ(span_log.GetTagOptional("grpc_code"), "OK") << span_log; +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/src/logging_test.cpp b/grpc/tests/src/logging_test.cpp index 4429a57252d2..974dbf3326e9 100644 --- a/grpc/tests/src/logging_test.cpp +++ b/grpc/tests/src/logging_test.cpp @@ -70,7 +70,7 @@ UTEST_F(GrpcAccessLog, Test) { R"(grpc_status=\d+\t)" R"(grpc_status_code=[A-Z_]+\n)"; - const auto logs = member.ExtractSingle(); + const auto logs = GetSingleLog(member.GetAll()); EXPECT_TRUE( utils::regex_match(logs.GetLogRaw(), utils::regex(kExpectedPattern))) << logs; diff --git a/grpc/utest/src/ugrpc/tests/service.cpp b/grpc/utest/src/ugrpc/tests/service.cpp index 29319e422f91..9e007b95b6f0 100644 --- a/grpc/utest/src/ugrpc/tests/service.cpp +++ b/grpc/utest/src/ugrpc/tests/service.cpp @@ -15,8 +15,7 @@ namespace ugrpc::tests { namespace { using ClientLogMiddlewareFactory = client::middlewares::log::MiddlewareFactory; -using ClientLogMiddlewareSettings = - client::middlewares::log::Middleware::Settings; +using ClientLogMiddlewareSettings = client::middlewares::log::Settings; using ClientDpMiddlewareFactory = client::middlewares::deadline_propagation::MiddlewareFactory; diff --git a/scripts/grpc/templates/client.usrv.cpp.jinja b/scripts/grpc/templates/client.usrv.cpp.jinja index 572c3383159c..77e05eaa9081 100644 --- a/scripts/grpc/templates/client.usrv.cpp.jinja +++ b/scripts/grpc/templates/client.usrv.cpp.jinja @@ -57,12 +57,12 @@ constexpr const auto& k{{service.name}}ClientQosConfig = std::unique_ptr<::grpc::ClientContext> context, const USERVER_NAMESPACE::ugrpc::client::Qos& qos ) const { + auto& stub = impl_.NextStub<{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}>(); return { USERVER_NAMESPACE::ugrpc::client::impl::CreateCallParams( impl_, {{method_id}}, std::move(context), k{{service.name}}ClientQosConfig, qos ), - impl_.NextStub<{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}>(), - &{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}::Stub::PrepareAsync{{method.name}}, + [&stub](auto&&... args) { return stub.PrepareAsync{{method.name}}(std::forward(args)...); }, {% if method.client_streaming %} }; {% else %} diff --git a/scripts/grpc/templates/client.usrv.hpp.jinja b/scripts/grpc/templates/client.usrv.hpp.jinja index 781bcc4dd322..e4d42a496ef1 100644 --- a/scripts/grpc/templates/client.usrv.hpp.jinja +++ b/scripts/grpc/templates/client.usrv.hpp.jinja @@ -52,11 +52,11 @@ class {{service.name}}Client final { static USERVER_NAMESPACE::ugrpc::impl::StaticServiceMetadata GetMetadata(); private: - {# All non-template related client data should be placed in ClientData #} template friend USERVER_NAMESPACE::ugrpc::client::impl::ClientData& USERVER_NAMESPACE::ugrpc::client::impl::GetClientData(Client& client); + {# All non-template related client data should be placed in ClientData #} USERVER_NAMESPACE::ugrpc::client::impl::ClientData impl_; }; {% endfor %} diff --git a/universal/include/userver/utils/impl/source_location.hpp b/universal/include/userver/utils/impl/source_location.hpp index 71234c7c8ec7..061933647688 100644 --- a/universal/include/userver/utils/impl/source_location.hpp +++ b/universal/include/userver/utils/impl/source_location.hpp @@ -95,6 +95,8 @@ class SourceLocation final { std::string_view function_name_; }; +std::string ToString(const SourceLocation& location); + } // namespace utils::impl USERVER_NAMESPACE_END diff --git a/universal/include/userver/utils/span.hpp b/universal/include/userver/utils/span.hpp index 9545a6d1b779..c075c1569ee1 100644 --- a/universal/include/userver/utils/span.hpp +++ b/universal/include/userver/utils/span.hpp @@ -17,6 +17,18 @@ USERVER_NAMESPACE_BEGIN namespace utils { +namespace impl { + +template +struct TypeIdentityImpl final { + using type = T; +}; + +template +using TypeIdentity = typename TypeIdentityImpl::type; + +} // namespace impl + /// A polyfill for std::span from C++20 template class span final { @@ -48,7 +60,11 @@ class span final { T*>>> // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) constexpr /*implicit*/ span(Container&& cont) noexcept - : span(std::data(cont), std::data(cont) + std::size(cont)) {} + : span(std::data(cont), std::size(cont)) {} + + template + constexpr /*implicit*/ span(impl::TypeIdentity (&array)[Size]) noexcept + : span(std::data(array), std::size(array)) {} constexpr T* begin() const noexcept { return begin_; } constexpr T* end() const noexcept { return end_; } @@ -72,8 +88,8 @@ class span final { return span{begin_ + offset, end_}; } - constexpr span subspan(std::size_t offset, std::size_t count) const - noexcept { + constexpr span subspan(std::size_t offset, // + std::size_t count) const noexcept { UASSERT(offset + count <= size()); return span{begin_ + offset, begin_ + offset + count}; } diff --git a/universal/src/utils/impl/source_location.cpp b/universal/src/utils/impl/source_location.cpp new file mode 100644 index 000000000000..92d71cf097b5 --- /dev/null +++ b/universal/src/utils/impl/source_location.cpp @@ -0,0 +1,16 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace utils::impl { + +std::string ToString(const SourceLocation& location) { + return StrCat(location.GetFunctionName(), " (", location.GetFileName(), ":", + location.GetLineString(), ")"); +} + +} // namespace utils::impl + +USERVER_NAMESPACE_END diff --git a/universal/utest/include/userver/utest/log_capture_fixture.hpp b/universal/utest/include/userver/utest/log_capture_fixture.hpp index 742395fa64b5..baf302ee2e0e 100644 --- a/universal/utest/include/userver/utest/log_capture_fixture.hpp +++ b/universal/utest/include/userver/utest/log_capture_fixture.hpp @@ -4,6 +4,7 @@ /// @brief @copybrief utest::LogCaptureFixture #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -42,7 +44,10 @@ class LogRecord final { const std::string& GetTag(std::string_view key) const; /// @returns decoded value of the tag in the log record, or `std::nullopt` - const std::string* GetTagOptional(std::string_view key) const; + std::optional GetTagOptional(std::string_view key) const; + + /// @returns decoded value of the tag in the log record, or `nullptr` + const std::string* GetTagOrNullptr(std::string_view key) const; /// @returns serialized log record const std::string& GetLogRaw() const; @@ -66,6 +71,18 @@ std::ostream& operator<<(std::ostream&, const LogRecord& data); std::ostream& operator<<(std::ostream&, const std::vector& data); +/// Thrown by @ref GetSingleLog. +class NotSingleLogError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +/// @returns the only log record from `log`. +/// @throws NotSingleLogError if there are zero or multiple log records. +LogRecord GetSingleLog(utils::span log, + const utils::impl::SourceLocation& source_location = + utils::impl::SourceLocation::Current()); + /// @brief A mocked logger that stores the log records in memory. /// @see @ref utest::LogCaptureFixture class LogCaptureLogger final { @@ -76,20 +93,19 @@ class LogCaptureLogger final { logging::LoggerPtr GetLogger() const; /// @returns all collected logs. + /// @see @ref GetSingleLog std::vector GetAll() const; - /// @returns the single collected log record, then clears logs. - /// @throws std::runtime_error if there are zero or multiple log records. - LogRecord ExtractSingle(); - /// @returns logs filtered by (optional) text substring and (optional) tags /// substrings. + /// @see @ref GetSingleLog std::vector Filter( std::string_view text_substring, - utils::span> + utils::span> tag_substrings = {}) const; /// @returns logs filtered by an arbitrary predicate. + /// @see @ref GetSingleLog std::vector Filter( utils::function_ref predicate) const; @@ -101,7 +117,9 @@ class LogCaptureLogger final { std::string ToStringViaLogging(const T& value) { Clear(); LOG_CRITICAL() << value; - return ExtractSingle().GetText(); + auto text = GetSingleLog(GetAll()).GetText(); + Clear(); + return text; } private: diff --git a/universal/utest/src/utest/log_capture_fixture.cpp b/universal/utest/src/utest/log_capture_fixture.cpp index 2d1d5036fad0..153cd053e4b5 100644 --- a/universal/utest/src/utest/log_capture_fixture.cpp +++ b/universal/utest/src/utest/log_capture_fixture.cpp @@ -15,6 +15,8 @@ USERVER_NAMESPACE_BEGIN namespace utest { +namespace {} // namespace + namespace impl { class ToStringLogger : public logging::impl::LoggerBase { @@ -64,7 +66,7 @@ class ToStringLogger : public logging::impl::LoggerBase { const std::string& LogRecord::GetText() const { return GetTag("text"); } const std::string& LogRecord::GetTag(std::string_view key) const { - auto tag_value = GetTagOptional(key); + auto tag_value = GetTagOrNullptr(key); if (!tag_value) { throw std::runtime_error( fmt::format("No '{}' tag in log record:\n{}", key, log_raw_)); @@ -72,7 +74,15 @@ const std::string& LogRecord::GetTag(std::string_view key) const { return *std::move(tag_value); } -const std::string* LogRecord::GetTagOptional(std::string_view key) const { +std::optional LogRecord::GetTagOptional( + std::string_view key) const { + if (const auto* const value = GetTagOrNullptr(key)) { + return *value; + } + return std::nullopt; +} + +const std::string* LogRecord::GetTagOrNullptr(std::string_view key) const { const auto iter = std::find_if(tags_.begin(), tags_.end(), [&](const std::pair& tag) { @@ -131,6 +141,21 @@ std::ostream& operator<<(std::ostream& os, const std::vector& data) { return os; } +LogRecord GetSingleLog(utils::span log, + const utils::impl::SourceLocation& source_location) { + if (log.size() != 1) { + std::string msg = + fmt::format("There are {} log records instead of 1 at {}:\n", + log.size(), ToString(source_location)); + for (const auto& record : log) { + msg += record.GetLogRaw(); + } + throw NotSingleLogError(msg); + } + auto single_record = std::move(log[0]); + return single_record; +} + LogCaptureLogger::LogCaptureLogger(logging::Format format) : logger_(utils::MakeSharedRef(format)) {} @@ -142,29 +167,14 @@ std::vector LogCaptureLogger::GetAll() const { return logger_->GetAll(); } -LogRecord LogCaptureLogger::ExtractSingle() { - auto log = GetAll(); - if (log.size() != 1) { - std::string msg = - fmt::format("There are {} log records instead of 1:\n", log.size()); - for (const auto& record : log) { - msg += record.GetLogRaw(); - } - throw std::runtime_error(msg); - } - auto single_record = std::move(log[0]); - Clear(); - return single_record; -} - std::vector LogCaptureLogger::Filter( std::string_view text_substring, - utils::span> tag_substrings) - const { + utils::span> + tag_substrings) const { return Filter([&](const LogRecord& record) { return record.GetText().find(text_substring) != std::string_view::npos && boost::algorithm::all_of(tag_substrings, [&](const auto& kv) { - const auto* tag_value = record.GetTagOptional(kv.first); + const auto* tag_value = record.GetTagOrNullptr(kv.first); return tag_value && tag_value->find(kv.second) != std::string_view::npos; }); diff --git a/ydb/tests/small_table.hpp b/ydb/tests/small_table.hpp index 968123d27ebc..6e1b09798773 100644 --- a/ydb/tests/small_table.hpp +++ b/ydb/tests/small_table.hpp @@ -56,11 +56,6 @@ inline const std::vector kPreFilledRows = { {"key3", "value3", 3}, }; -inline std::string ToString(const utils::impl::SourceLocation& location) { - return fmt::format("at {}, {}:{}", location.GetFunctionName(), - location.GetFileName(), location.GetLineString()); -} - template void AssertNullableColumn(ydb::Row& row, std::string_view column_name, const T& expected, @@ -89,7 +84,7 @@ inline auto AssertArePreFilledRows(ydb::Cursor cursor, const utils::impl::SourceLocation& location = utils::impl::SourceLocation::Current()) { ASSERT_THAT(cursor, testing::SizeIs(indexes.size())) - << "expected " << indexes.size() << " rows in cursor " + << "expected " << indexes.size() << " rows in cursor at " << ToString(location); for (auto [pos, row] : utils::enumerate(cursor)) {