From c04959e2a164c78b6dc8597d40ee9981a4758f82 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Wed, 20 Jul 2022 17:20:33 +0200 Subject: [PATCH 01/17] added RPC system as enum --- .../src/opentelemetry/instrumentation/grpc/_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 82e192622c..30d81a91a0 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -29,7 +29,7 @@ from opentelemetry import trace from opentelemetry.context import attach, detach from opentelemetry.propagate import extract -from opentelemetry.semconv.trace import SpanAttributes +from opentelemetry.semconv.trace import RpcSystemValues, SpanAttributes from opentelemetry.trace.status import Status, StatusCode logger = logging.getLogger(__name__) @@ -206,7 +206,7 @@ def _start_span( # standard attributes attributes = { - SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_SYSTEM: RpcSystemValues.GRPC.value, SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], } @@ -217,8 +217,8 @@ def _start_span( ) attributes.update( { - SpanAttributes.RPC_METHOD: method, SpanAttributes.RPC_SERVICE: service, + SpanAttributes.RPC_METHOD: method, } ) From 5dd106a6b27594ef31345985435a73ad97536199 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Wed, 20 Jul 2022 17:24:34 +0200 Subject: [PATCH 02/17] added support of code() and details() --- .../instrumentation/grpc/_server.py | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 30d81a91a0..5731af6d6e 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -68,8 +68,8 @@ class _OpenTelemetryServicerContext(grpc.ServicerContext): def __init__(self, servicer_context, active_span): self._servicer_context = servicer_context self._active_span = active_span - self.code = grpc.StatusCode.OK - self.details = None + self._code = grpc.StatusCode.OK + self._details = None super().__init__() def __getattr__(self, attr): @@ -118,26 +118,41 @@ def trailing_metadata(self): return self._servicer_context.trailing_metadata() def abort(self, code, details): - self.code = code - self.details = details + if not hasattr(self._servicer_context, "abort"): + raise RuntimeError( + "abort() is not supported with the installed version of grpcio" + ) + self._code = code + self._details = details self._active_span.set_attribute( SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] ) self._active_span.set_status( Status( status_code=StatusCode.ERROR, - description=f"{code}:{details}", + description=f"{code}: {details}", ) ) return self._servicer_context.abort(code, details) def abort_with_status(self, status): + if not hasattr(self._servicer_context, "abort_with_status"): + raise RuntimeError( + "abort_with_status() is not supported with the installed version of grpcio" + ) return self._servicer_context.abort_with_status(status) + def code(self): + if not hasattr(self._servicer_context, "code"): + raise RuntimeError( + "code() is not supported with the installed version of grpcio" + ) + return self._servicer_context.code() + def set_code(self, code): - self.code = code + self._code = code # use details if we already have it, otherwise the status description - details = self.details or code.value[1] + details = self._details or code.value[1] self._active_span.set_attribute( SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] ) @@ -145,18 +160,26 @@ def set_code(self, code): self._active_span.set_status( Status( status_code=StatusCode.ERROR, - description=f"{code}:{details}", + description=f"{code}: {details}", ) ) return self._servicer_context.set_code(code) + def details(self): + if not hasattr(self._servicer_context, "details"): + raise RuntimeError( + "details() is not supported with the installed version of " + "grpcio" + ) + return self._servicer_context.details() + def set_details(self, details): - self.details = details - if self.code != grpc.StatusCode.OK: + self._details = details + if self._code != grpc.StatusCode.OK: self._active_span.set_status( Status( status_code=StatusCode.ERROR, - description=f"{self.code}:{details}", + description=f"{self._code}: {details}", ) ) return self._servicer_context.set_details(details) From 54be1cf6db82d2ee58e44a165eabc493d708a327 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Wed, 20 Jul 2022 20:58:50 +0200 Subject: [PATCH 03/17] added setting of code and details in case of cancel --- .../src/opentelemetry/instrumentation/grpc/_server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 5731af6d6e..9329eb7077 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -82,6 +82,17 @@ def time_remaining(self, *args, **kwargs): return self._servicer_context.time_remaining(*args, **kwargs) def cancel(self, *args, **kwargs): + self._code = grpc.StatusCode.CANCELLED + self._details = self._code.value[1] + self._active_span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, self._code.value[0] + ) + self._active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{self._code}: {self._details}", + ) + ) return self._servicer_context.cancel(*args, **kwargs) def add_callback(self, *args, **kwargs): From c614a87693229c68a091e6ce53f2e7feed3b8635 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Fri, 22 Jul 2022 15:18:06 +0200 Subject: [PATCH 04/17] added meter instrumentation --- .../instrumentation/grpc/__init__.py | 34 ++- .../instrumentation/grpc/_server.py | 288 +++++++++++++----- 2 files changed, 239 insertions(+), 83 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py index f6de2a2d76..f67f938d53 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py @@ -67,8 +67,13 @@ def run(): import grpc - from opentelemetry import trace + from opentelemetry import metrics, trace from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( ConsoleSpanExporter, @@ -80,6 +85,12 @@ def run(): except ImportError: from gen import helloworld_pb2, helloworld_pb2_grpc + exporter = ConsoleMetricExporter() + reader = PeriodicExportingMetricReader(exporter) + metrics.set_meter_provider( + MeterProvider(metric_readers=[reader]) + ) + trace.set_tracer_provider(TracerProvider()) trace.get_tracer_provider().add_span_processor( SimpleSpanProcessor(ConsoleSpanExporter()) @@ -123,7 +134,7 @@ def serve(): import grpc # pylint:disable=import-self from wrapt import wrap_function_wrapper as _wrap -from opentelemetry import trace +from opentelemetry import metrics, trace from opentelemetry.instrumentation.grpc.grpcext import intercept_channel from opentelemetry.instrumentation.grpc.package import _instruments from opentelemetry.instrumentation.grpc.version import __version__ @@ -153,17 +164,24 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs): self._original_func = grpc.server + meter_provider = kwargs.get("meter_provider") tracer_provider = kwargs.get("tracer_provider") def server(*args, **kwargs): if "interceptors" in kwargs: # add our interceptor as the first kwargs["interceptors"].insert( - 0, server_interceptor(tracer_provider=tracer_provider) + 0, server_interceptor( + meter_provider=meter_provider, + tracer_provider=tracer_provider + ) ) else: kwargs["interceptors"] = [ - server_interceptor(tracer_provider=tracer_provider) + server_interceptor( + meter_provider=meter_provider, + tracer_provider=tracer_provider + ) ] return self._original_func(*args, **kwargs) @@ -240,17 +258,19 @@ def client_interceptor(tracer_provider=None): return _client.OpenTelemetryClientInterceptor(tracer) -def server_interceptor(tracer_provider=None): +def server_interceptor(meter_provider=None, tracer_provider=None): """Create a gRPC server interceptor. Args: - tracer: The tracer to use to create server-side spans. + meter_provider: The meter provider which allows acess to the meter. + tracer_provider: The tracer provider which allows acess to the tracer. Returns: A service-side interceptor object. """ from . import _server + meter = metrics.get_meter(__name__, __version__, meter_provider) tracer = trace.get_tracer(__name__, __version__, tracer_provider) - return _server.OpenTelemetryServerInterceptor(tracer) + return _server.OpenTelemetryServerInterceptor(meter, tracer) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 9329eb7077..1bcaeb776d 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -29,12 +29,21 @@ from opentelemetry import trace from opentelemetry.context import attach, detach from opentelemetry.propagate import extract -from opentelemetry.semconv.trace import RpcSystemValues, SpanAttributes +from opentelemetry.semconv.trace import MessageTypeValues, RpcSystemValues, SpanAttributes from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util._time import _time_ns + logger = logging.getLogger(__name__) +_MESSAGE = "message" +"""event name of a message.""" + +_RPC_USER_AGENT = "rpc.user_agent" +"""span attribute for RPC user agent.""" + + # wrap an RPC call # see https://github.com/grpc/grpc/issues/18191 def _wrap_rpc_behavior(handler, continuation): @@ -63,8 +72,25 @@ def _wrap_rpc_behavior(handler, continuation): ) +def _add_message_event( + span, + message_type, + message_size_by, + message_id=1 +): + span.add_event( + _MESSAGE, + { + SpanAttributes.MESSAGE_TYPE: message_type, + SpanAttributes.MESSAGE_ID: message_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: message_size_by, + } + ) + + # pylint:disable=abstract-method class _OpenTelemetryServicerContext(grpc.ServicerContext): + def __init__(self, servicer_context, active_span): self._servicer_context = servicer_context self._active_span = active_span @@ -83,7 +109,7 @@ def time_remaining(self, *args, **kwargs): def cancel(self, *args, **kwargs): self._code = grpc.StatusCode.CANCELLED - self._details = self._code.value[1] + self._details = grpc.StatusCode.CANCELLED.value[1] self._active_span.set_attribute( SpanAttributes.RPC_GRPC_STATUS_CODE, self._code.value[0] ) @@ -101,8 +127,8 @@ def add_callback(self, *args, **kwargs): def disable_next_message_compression(self): return self._service_context.disable_next_message_compression() - def invocation_metadata(self, *args, **kwargs): - return self._servicer_context.invocation_metadata(*args, **kwargs) + def invocation_metadata(self): + return self._servicer_context.invocation_metadata() def peer(self): return self._servicer_context.peer() @@ -205,24 +231,53 @@ class OpenTelemetryServerInterceptor(grpc.ServerInterceptor): Usage:: + meter = some OpenTelemetry meter tracer = some OpenTelemetry tracer interceptors = [ - OpenTelemetryServerInterceptor(tracer), + OpenTelemetryServerInterceptor(meter, tracer), ] server = grpc.server( futures.ThreadPoolExecutor(max_workers=concurrency), - interceptors = interceptors) + interceptors = interceptors + ) """ - def __init__(self, tracer): + def __init__(self, meter, tracer): + self._meter = meter self._tracer = tracer + self._duration_histogram = self._meter.create_histogram( + name="rpc.server.duration", + unit="ms", + description="measures the duration of the inbound rpc", + ) + self._request_size_histogram = self._meter.create_histogram( + name="rpc.server.request.size", + unit="By", + description="measures size of RPC request messages (uncompressed)", + ) + self._response_size_histogram = self._meter.create_histogram( + name="rpc.server.response.size", + unit="By", + description="measures size of RPC response messages (uncompressed)", + ) + self._requests_per_rpc_histogram = self._meter.create_histogram( + name="rpc.server.requests_per_rpc", + unit="requests", + description="measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs", + ) + self._responses_per_rpc_histogram = self._meter.create_histogram( + name="rpc.server.responses_per_rpc", + unit="responses", + description="measures the number of messages sent per RPC. Should be 1 for all non-streaming RPCs", + ) + @contextmanager - def _set_remote_context(self, servicer_context): - metadata = servicer_context.invocation_metadata() + def _set_remote_context(self, context): + metadata = context.invocation_metadata() if metadata: md_dict = {md.key: md.value for md in metadata} ctx = extract(md_dict) @@ -234,32 +289,26 @@ def _set_remote_context(self, servicer_context): else: yield - def _start_span( - self, handler_call_details, context, set_status_on_exception=False - ): - + def _create_attributes(self, context, full_method): # standard attributes attributes = { SpanAttributes.RPC_SYSTEM: RpcSystemValues.GRPC.value, SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], } - # if we have details about the call, split into service and method - if handler_call_details.method: - service, method = handler_call_details.method.lstrip("/").split( - "/", 1 - ) - attributes.update( - { - SpanAttributes.RPC_SERVICE: service, - SpanAttributes.RPC_METHOD: method, - } - ) + # add service and method attributes + service, method = full_method.lstrip("/").split("/", 1) + attributes.update( + { + SpanAttributes.RPC_SERVICE: service, + SpanAttributes.RPC_METHOD: method, + } + ) # add some attributes from the metadata metadata = dict(context.invocation_metadata()) if "user-agent" in metadata: - attributes["rpc.user_agent"] = metadata["user-agent"] + attributes[_RPC_USER_AGENT] = metadata["user-agent"] # Split up the peer to keep with how other telemetry sources # do it. This looks like: @@ -285,72 +334,159 @@ def _start_span( except IndexError: logger.warning("Failed to parse peer address '%s'", context.peer()) - return self._tracer.start_as_current_span( - name=handler_call_details.method, - kind=trace.SpanKind.SERVER, - attributes=attributes, - set_status_on_exception=set_status_on_exception, - ) + return attributes def intercept_service(self, continuation, handler_call_details): def telemetry_wrapper(behavior, request_streaming, response_streaming): def telemetry_interceptor(request_or_iterator, context): - - # handle streaming responses specially - if response_streaming: - return self._intercept_server_stream( - behavior, - handler_call_details, - request_or_iterator, - context, - ) - with self._set_remote_context(context): - with self._start_span( - handler_call_details, - context, - set_status_on_exception=False, + attributes = self._create_attributes(context, handler_call_details.method) + + with self._tracer.start_as_current_span( + name=handler_call_details.method, + kind=trace.SpanKind.SERVER, + attributes=attributes, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False ) as span: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - # And now we run the actual RPC. try: - return behavior(request_or_iterator, context) - - except Exception as error: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # wrap / log the request (iterator) + if request_streaming: + request_or_iterator = self._log_stream_requests( + request_or_iterator, span, attributes + ) + else: + self._log_unary_request( + request_or_iterator, span, attributes + ) + + # call the actual RPC and track the duration + with self._record_duration(attributes, context): + response_or_iterator = behavior(request_or_iterator, context) + + # wrap / log the response (iterator) + if response_streaming: + response_or_iterator = self._log_stream_responses( + response_or_iterator, span, attributes, context + ) + else: + self._log_unary_response( + response_or_iterator, span, attributes, context + ) + + return response_or_iterator + + except Exception as exc: # Bare exceptions are likely to be gRPC aborts, which # we handle in our context wrapper. # Here, we're interested in uncaught exceptions. # pylint:disable=unidiomatic-typecheck - if type(error) != Exception: - span.record_exception(error) - raise error + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + + finally: + if not response_streaming: + span.end() return telemetry_interceptor return _wrap_rpc_behavior( - continuation(handler_call_details), telemetry_wrapper + continuation(handler_call_details), + telemetry_wrapper + ) + + def _log_unary_request(self, request, active_span, attributes): + message_size_by = request.ByteSize() + _add_message_event( + active_span, MessageTypeValues.RECEIVED.value, message_size_by + ) + self._request_size_histogram.record(message_size_by, attributes) + self._requests_per_rpc_histogram.record(1, attributes) + + def _log_unary_response(self, response, active_span, attributes, context): + message_size_by = response.ByteSize() + _add_message_event( + active_span, MessageTypeValues.SENT.value, message_size_by ) + if context._code != grpc.StatusCode.OK: + attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] + self._response_size_histogram.record(message_size_by, attributes) + self._responses_per_rpc_histogram.record(1, attributes) + + def _log_stream_requests(self, request_iterator, active_span, attributes): + req_id = 1 + for req_id, msg in enumerate(request_iterator, start=1): + message_size_by = msg.ByteSize() + _add_message_event( + active_span, MessageTypeValues.RECEIVED.value, message_size_by, message_id=req_id + ) + self._request_size_histogram.record(message_size_by, attributes) + yield msg + + self._requests_per_rpc_histogram.record(req_id, attributes) + + def _log_stream_responses(self, response_iterator, active_span, attributes, context): + with trace.use_span( + active_span, + end_on_exit=True, + record_exception=False, + set_status_on_exception=False + ): + try: + res_id = 1 + for res_id, msg in enumerate(response_iterator, start=1): + message_size_by = msg.ByteSize() + _add_message_event( + active_span, MessageTypeValues.SENT.value, message_size_by, message_id=res_id + ) + self._response_size_histogram.record(message_size_by, attributes) + yield msg + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + active_span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + active_span.record_exception(exc) + raise exc + finally: + if context._code != grpc.StatusCode.OK: + attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] + self._responses_per_rpc_histogram.record(res_id, attributes) - # Handle streaming responses separately - we have to do this - # to return a *new* generator or various upstream things - # get confused, or we'll lose the consistent trace - def _intercept_server_stream( - self, behavior, handler_call_details, request_or_iterator, context - ): - - with self._set_remote_context(context): - with self._start_span( - handler_call_details, context, set_status_on_exception=False - ) as span: - context = _OpenTelemetryServicerContext(context, span) - - try: - yield from behavior(request_or_iterator, context) - - except Exception as error: - # pylint:disable=unidiomatic-typecheck - if type(error) != Exception: - span.record_exception(error) - raise error + @contextmanager + def _record_duration(self, attributes, context): + start = _time_ns() + try: + yield + finally: + duration = max(round((_time_ns() - start) * 1000), 0) + if context._code != grpc.StatusCode.OK: + attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] + self._duration_histogram.record(duration, attributes) From e4bf609c03eeadfe77038d80421df6e8b7b23bbe Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Fri, 22 Jul 2022 15:18:40 +0200 Subject: [PATCH 05/17] added test of events and test cases for streaming abort, cancel, error and raised exception --- .../tests/test_server_interceptor.py | 1050 +++++++++++++++-- 1 file changed, 977 insertions(+), 73 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py index 03e063aa0d..b0669032ca 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py @@ -38,26 +38,6 @@ ) -class UnaryUnaryMethodHandler(grpc.RpcMethodHandler): - def __init__(self, handler): - self.request_streaming = False - self.response_streaming = False - self.request_deserializer = None - self.response_serializer = None - self.unary_unary = handler - self.unary_stream = None - self.stream_unary = None - self.stream_stream = None - - -class UnaryUnaryRpcHandler(grpc.GenericRpcHandler): - def __init__(self, handler): - self._unary_unary_handler = handler - - def service(self, handler_call_details): - return UnaryUnaryMethodHandler(self._unary_unary_handler) - - class Servicer(GRPCTestServerServicer): """Our test servicer""" @@ -78,9 +58,15 @@ def ServerStreamingMethod(self, request, context): class TestOpenTelemetryServerInterceptor(TestBase): + # pylint:disable=C0103 + + def assertEvent(self, event, name, attributes): + self.assertEqual(event.name, name) + for key, val in attributes.items(): + self.assertIn(key, event.attributes, msg=str(event.attributes)) + self.assertEqual(val, event.attributes[key], msg=str(event.attributes)) + def test_instrumentor(self): - def handler(request, context): - return b"" grpc_server_instrumentor = GrpcInstrumentorServer() grpc_server_instrumentor.instrument() @@ -89,16 +75,17 @@ def handler(request, context): executor, options=(("grpc.so_reuseport", 0),), ) - - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) - + add_GRPCTestServerServicer_to_server(Servicer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") - rpc_call = "TestServicer/handler" + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + try: server.start() - channel.unary_unary(rpc_call)(b"test") + response = channel.unary_unary(rpc_call)(msg) finally: server.stop(None) @@ -113,14 +100,15 @@ def handler(request, context): span, opentelemetry.instrumentation.grpc ) - # Check attributes + # check attributes self.assertSpanHasAttributes( span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", - SpanAttributes.RPC_METHOD: "handler", - SpanAttributes.RPC_SERVICE: "TestServicer", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ 0 @@ -128,11 +116,30 @@ def handler(request, context): }, ) + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + grpc_server_instrumentor.uninstrument() def test_uninstrument(self): - def handler(request, context): - return b"" grpc_server_instrumentor = GrpcInstrumentorServer() grpc_server_instrumentor.instrument() @@ -142,16 +149,16 @@ def handler(request, context): executor, options=(("grpc.so_reuseport", 0),), ) - - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) - + add_GRPCTestServerServicer_to_server(Servicer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") - rpc_call = "TestServicer/test" + rpc_call = "/GRPCTestServer/SimpleMethod" + msg = Request().SerializeToString() + try: server.start() - channel.unary_unary(rpc_call)(b"test") + channel.unary_unary(rpc_call)(msg) finally: server.stop(None) @@ -177,9 +184,10 @@ def test_create_span(self): rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") msg = request.SerializeToString() + try: server.start() - channel.unary_unary(rpc_call)(msg) + response = channel.unary_unary(rpc_call)(msg) finally: server.stop(None) @@ -200,6 +208,7 @@ def test_create_span(self): span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", @@ -210,6 +219,27 @@ def test_create_span(self): }, ) + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + def test_create_two_spans(self): """Verify that the interceptor captures sub spans within the given trace""" @@ -246,9 +276,10 @@ def SimpleMethod(self, request, context): rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") msg = request.SerializeToString() + try: server.start() - channel.unary_unary(rpc_call)(msg) + response = channel.unary_unary(rpc_call)(msg) finally: server.stop(None) @@ -260,16 +291,17 @@ def SimpleMethod(self, request, context): self.assertEqual(parent_span.name, rpc_call) self.assertIs(parent_span.kind, trace.SpanKind.SERVER) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( parent_span, opentelemetry.instrumentation.grpc ) - # Check attributes + # check parent attributes self.assertSpanHasAttributes( parent_span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", @@ -280,12 +312,41 @@ def SimpleMethod(self, request, context): }, ) - # Check the child span + # check parent events + self.assertEqual(len(parent_span.events), 2) + self.assertEvent( + parent_span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + parent_span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + + # check the child span self.assertEqual(child_span.name, "child") self.assertEqual( parent_span.context.trace_id, child_span.context.trace_id ) + # check child event + self.assertEqual(len(child_span.events), 1) + self.assertEvent( + child_span.events[0], + "child event", + {} + ) + def test_create_span_streaming(self): """Check that the interceptor wraps calls with spans server-side, on a streaming call.""" @@ -307,9 +368,10 @@ def test_create_span_streaming(self): rpc_call = "/GRPCTestServer/ServerStreamingMethod" request = Request(client_id=1, request_data="test") msg = request.SerializeToString() + try: server.start() - list(channel.unary_stream(rpc_call)(msg)) + responses = list(channel.unary_stream(rpc_call)(msg)) finally: server.stop(None) @@ -320,16 +382,17 @@ def test_create_span_streaming(self): self.assertEqual(span.name, rpc_call) self.assertIs(span.kind, trace.SpanKind.SERVER) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) - # Check attributes + # check attributes self.assertSpanHasAttributes( span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", @@ -340,6 +403,30 @@ def test_create_span_streaming(self): }, ) + # check events + self.assertEqual(len(span.events), len(responses) + 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + for res_id, (event, response) in enumerate( + zip(span.events[1:], responses), start=1 + ): + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + def test_create_two_spans_streaming(self): """Verify that the interceptor captures sub spans in a streaming call, within the given trace""" @@ -376,6 +463,7 @@ def ServerStreamingMethod(self, request, context): rpc_call = "/GRPCTestServer/ServerStreamingMethod" request = Request(client_id=1, request_data="test") msg = request.SerializeToString() + try: server.start() list(channel.unary_stream(rpc_call)(msg)) @@ -400,6 +488,7 @@ def ServerStreamingMethod(self, request, context): parent_span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", @@ -427,7 +516,10 @@ def test_span_lifetime(self): def handler(request, context): nonlocal active_span_in_handler active_span_in_handler = trace.get_current_span() - return b"" + return Response() + + servicer = Servicer() + servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( @@ -435,15 +527,64 @@ def handler(request, context): options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + msg = Request().SerializeToString() + active_span_before_call = trace.get_current_span() + try: + server.start() + channel.unary_unary(rpc_call)(msg) + finally: + server.stop(None) + active_span_after_call = trace.get_current_span() + + self.assertEqual(active_span_before_call, trace.INVALID_SPAN) + self.assertEqual(active_span_after_call, trace.INVALID_SPAN) + self.assertIsInstance(active_span_in_handler, trace_sdk.Span) + self.assertIsNone(active_span_in_handler.parent) + + def test_span_lifetime_streaming(self): + """Check that the span is active for the duration of the call.""" + + interceptor = server_interceptor() + + # To capture the current span at the time the handler is called + active_span_in_handler = None + + def handler(request, context): + nonlocal active_span_in_handler + active_span_in_handler = trace.get_current_span() + for data in ("one", "two", "three"): + yield Response( + server_id=request.client_id, + response_data=data, + ) + + servicer = Servicer() + servicer.ServerStreamingMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + active_span_before_call = trace.get_current_span() try: server.start() - channel.unary_unary("TestServicer/handler")(b"") + list(channel.unary_stream(rpc_call)(msg)) finally: server.stop(None) active_span_after_call = trace.get_current_span() @@ -463,7 +604,10 @@ def test_sequential_server_spans(self): def handler(request, context): active_spans_in_handler.append(trace.get_current_span()) - return b"" + return Response() + + servicer = Servicer() + servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( @@ -471,15 +615,17 @@ def handler(request, context): options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) - + add_GRPCTestServerServicer_to_server(servicer, server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") + rpc_call = "/GRPCTestServer/SimpleMethod" + msg = Request().SerializeToString() + try: server.start() - channel.unary_unary("TestServicer/handler")(b"") - channel.unary_unary("TestServicer/handler")(b"") + response_1 = channel.unary_unary(rpc_call)(msg) + response_2 = channel.unary_unary(rpc_call)(msg) finally: server.stop(None) @@ -490,7 +636,7 @@ def handler(request, context): self.assertNotEqual(span1.context.span_id, span2.context.span_id) self.assertNotEqual(span1.context.trace_id, span2.context.trace_id) - for span in (span1, span2): + for span, response in zip([span1, span2], [response_1, response_2]): # each should be a root span self.assertIsNone(span2.parent) @@ -499,9 +645,10 @@ def handler(request, context): span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", - SpanAttributes.RPC_METHOD: "handler", - SpanAttributes.RPC_SERVICE: "TestServicer", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ 0 @@ -509,6 +656,26 @@ def handler(request, context): }, ) + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + def test_concurrent_server_spans(self): """Check that concurrent RPC calls don't interfere with each other. @@ -527,7 +694,10 @@ def test_concurrent_server_spans(self): def handler(request, context): latch() active_spans_in_handler.append(trace.get_current_span()) - return b"" + return Response() + + servicer = Servicer() + servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=2) as executor: server = grpc.server( @@ -535,21 +705,25 @@ def handler(request, context): options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) + add_GRPCTestServerServicer_to_server(servicer, server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + try: server.start() # Interleave calls so spans are active on each thread at the same # time with futures.ThreadPoolExecutor(max_workers=2) as tpe: f1 = tpe.submit( - channel.unary_unary("TestServicer/handler"), b"" + channel.unary_unary(rpc_call), msg ) f2 = tpe.submit( - channel.unary_unary("TestServicer/handler"), b"" + channel.unary_unary(rpc_call), msg ) futures.wait((f1, f2)) finally: @@ -562,18 +736,19 @@ def handler(request, context): self.assertNotEqual(span1.context.span_id, span2.context.span_id) self.assertNotEqual(span1.context.trace_id, span2.context.trace_id) - for span in (span1, span2): + for span, response in zip([span1, span2], [f1.result(), f2.result()]): # each should be a root span self.assertIsNone(span2.parent) - # check attributes + # Check attributes self.assertSpanHasAttributes( span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", - SpanAttributes.RPC_METHOD: "handler", - SpanAttributes.RPC_SERVICE: "TestServicer", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ 0 @@ -581,6 +756,27 @@ def handler(request, context): }, ) + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(response) + } + ) + def test_abort(self): """Check that we can catch an abort properly""" @@ -594,24 +790,116 @@ def test_abort(self): def handler(request, context): context.abort(grpc.StatusCode.FAILED_PRECONDITION, failure_message) + servicer = Servicer() + servicer.SimpleMethod = handler + with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + # unfortunately, these are just bare exceptions in grpc... + with self.assertRaises(Exception): + channel.unary_unary(rpc_call)(msg) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + ) + + # check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ + 0 + ], + }, + ) + + # check events + self.assertEqual(len(span.events), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + + def test_abort_streaming(self): + """Check that we can catch an abort of a streaming call properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # our detailed failure message + failure_message = "This is a test failure" + + # aborting RPC handler + def handler(request, context): + yield Response( + server_id=request.client_id, + response_data="one", + ) + context.abort(grpc.StatusCode.FAILED_PRECONDITION, failure_message) - server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler),)) + servicer = Servicer() + servicer.ServerStreamingMethod = handler + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") - rpc_call = "TestServicer/handler" + # setup the RPC + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() server.start() - # unfortunately, these are just bare exceptions in grpc... + responses = [] with self.assertRaises(Exception): - channel.unary_unary(rpc_call)(b"") + for res in channel.unary_stream(rpc_call)(msg): + responses.append(res) server.stop(None) spans_list = self.memory_exporter.get_finished_spans() @@ -621,7 +909,7 @@ def handler(request, context): self.assertEqual(span.name, rpc_call) self.assertIs(span.kind, trace.SpanKind.SERVER) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) @@ -630,17 +918,18 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{grpc.StatusCode.FAILED_PRECONDITION}:{failure_message}", + f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", ) - # Check attributes + # check attributes self.assertSpanHasAttributes( span, { SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", SpanAttributes.NET_PEER_NAME: "localhost", - SpanAttributes.RPC_METHOD: "handler", - SpanAttributes.RPC_SERVICE: "TestServicer", + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ 0 @@ -648,6 +937,621 @@ def handler(request, context): }, ) + # check events + self.assertEqual(len(span.events), 2) + self.assertEqual(len(responses), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(responses[0]) + } + ) + + def test_cancel(self): + """Check that we can catch a cancellation properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # aborting RPC handler + def handler(request, context): + context.cancel() + return Response() + + servicer = Servicer() + servicer.SimpleMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + # unfortunately, these are just bare exceptions in grpc... + with self.assertRaises(Exception): # as cm: + channel.unary_unary(rpc_call)(msg) + # exc = cm.exception + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: {grpc.StatusCode.CANCELLED.value[1]}", + ) + # self.assertEqual( + # span.status.description, + # f"{exc.code()}: {exc.details()}", + # ) + + # check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.CANCELLED.value[ + 0 + ], + }, + ) + + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: 0 + } + ) + + def test_cancel_streaming(self): + """Check that we can catch a cancellation of a streaming call properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # aborting RPC handler + def handler(request, context): + yield Response( + server_id=request.client_id, + response_data="one", + ) + context.cancel() + return + + servicer = Servicer() + servicer.ServerStreamingMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + responses = [] + with self.assertRaises(Exception): + for res in channel.unary_stream(rpc_call)(msg): + responses.append(res) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: {grpc.StatusCode.CANCELLED.value[1]}", + ) + # self.assertEqual( + # span.status.description, + # f"{exc.code()}: {exc.details()}", + # ) + + # check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.CANCELLED.value[ + 0 + ], + }, + ) + + # check events + self.assertEqual(len(span.events), 2) + self.assertEqual(len(responses), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(responses[0]) + } + ) + + def test_error(self): + """Check that we can catch an error properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # our detailed failure message + failure_message = "This is a test failure" + + # error RPC handler + def handler(request, context): + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(failure_message) + return Response() + + servicer = Servicer() + servicer.SimpleMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + # unfortunately, these are just bare exceptions in grpc... + with self.assertRaises(Exception): + channel.unary_unary(rpc_call)(msg) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # Check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + ) + + # Check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ + 0 + ], + }, + ) + + # Check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: 0 + } + ) + + def test_error_streaming(self): + """Check that we can catch an error in a streaming call properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # our detailed failure message + failure_message = "This is a test failure" + + # error RPC handler + def handler(request, context): + yield Response( + server_id=request.client_id, + response_data="one", + ) + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(failure_message) + return + + servicer = Servicer() + servicer.ServerStreamingMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + responses = [] + with self.assertRaises(Exception): + for res in channel.unary_stream(rpc_call)(msg): + responses.append(res) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # Check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + ) + + # Check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ + 0 + ], + }, + ) + + # Check events + self.assertEqual(len(span.events), 2) + self.assertEqual(len(responses), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(responses[0]) + } + ) + + def test_raise_exception(self): + """Check that we can catch a raised exception properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # our detailed error message + err_message = "This is a value error" + + # error RPC handler + def handler(request, context): + raise ValueError(err_message) + + servicer = Servicer() + servicer.SimpleMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + # unfortunately, these are just bare exceptions in grpc... + with self.assertRaises(Exception): + channel.unary_unary(rpc_call)(msg) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # Check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{ValueError.__name__}: {err_message}", + ) + + # Check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.UNKNOWN.value[ + 0 + ], + }, + ) + + # Check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": ValueError.__name__, + "exception.message": err_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_raise_exception_streaming(self): + """Check that we can catch a raised exception in a streaming call properly""" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + # our detailed error message + err_message = "This is a value error" + + # error RPC handler + def handler(request, context): + yield Response( + server_id=request.client_id, + response_data="one", + ) + raise ValueError(err_message) + + servicer = Servicer() + servicer.ServerStreamingMethod = handler + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(servicer, server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + msg = request.SerializeToString() + + server.start() + responses = [] + with self.assertRaises(Exception): + for res in channel.unary_stream(rpc_call)(msg): + responses.append(res) + server.stop(None) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, rpc_call) + self.assertIs(span.kind, trace.SpanKind.SERVER) + + # Check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{ValueError.__name__}: {err_message}", + ) + + # Check attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.UNKNOWN.value[ + 0 + ], + }, + ) + + # Check events + self.assertEqual(len(span.events), 3) + self.assertEqual(len(responses), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(responses[0]) + } + ) + self.assertEvent( + span.events[2], + "exception", + { + "exception.type": ValueError.__name__, + "exception.message": err_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + def get_latch(num): """Get a countdown latch function for use in n threads.""" From b3b9dedccb85c59589b3aedb9fa5b75bfa09980c Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Fri, 22 Jul 2022 17:11:16 +0200 Subject: [PATCH 06/17] bugfix of contextmanager of generator --- .../instrumentation/grpc/_server.py | 244 ++++++++++-------- 1 file changed, 139 insertions(+), 105 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 1bcaeb776d..4f9c23659d 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -252,7 +252,7 @@ def __init__(self, meter, tracer): self._duration_histogram = self._meter.create_histogram( name="rpc.server.duration", unit="ms", - description="measures the duration of the inbound rpc", + description="measures duration of inbound RPC", ) self._request_size_histogram = self._meter.create_histogram( name="rpc.server.request.size", @@ -339,70 +339,16 @@ def _create_attributes(self, context, full_method): def intercept_service(self, continuation, handler_call_details): def telemetry_wrapper(behavior, request_streaming, response_streaming): def telemetry_interceptor(request_or_iterator, context): - with self._set_remote_context(context): - attributes = self._create_attributes(context, handler_call_details.method) - - with self._tracer.start_as_current_span( - name=handler_call_details.method, - kind=trace.SpanKind.SERVER, - attributes=attributes, - end_on_exit=False, - record_exception=False, - set_status_on_exception=False - ) as span: - - try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - - # wrap / log the request (iterator) - if request_streaming: - request_or_iterator = self._log_stream_requests( - request_or_iterator, span, attributes - ) - else: - self._log_unary_request( - request_or_iterator, span, attributes - ) - - # call the actual RPC and track the duration - with self._record_duration(attributes, context): - response_or_iterator = behavior(request_or_iterator, context) - - # wrap / log the response (iterator) - if response_streaming: - response_or_iterator = self._log_stream_responses( - response_or_iterator, span, attributes, context - ) - else: - self._log_unary_response( - response_or_iterator, span, attributes, context - ) - - return response_or_iterator - - except Exception as exc: - # Bare exceptions are likely to be gRPC aborts, which - # we handle in our context wrapper. - # Here, we're interested in uncaught exceptions. - # pylint:disable=unidiomatic-typecheck - if type(exc) != Exception: - span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, - grpc.StatusCode.UNKNOWN.value[0] - ) - span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{type(exc).__name__}: {exc}", - ) - ) - span.record_exception(exc) - raise exc - - finally: - if not response_streaming: - span.end() + if response_streaming: + return self._intercept_streaming_response( + behavior, request_or_iterator, context, + request_streaming, handler_call_details.method + ) + + return self._intercept_unary_response( + behavior, request_or_iterator, context, + request_streaming, handler_call_details.method + ) return telemetry_interceptor @@ -411,6 +357,118 @@ def telemetry_interceptor(request_or_iterator, context): telemetry_wrapper ) + def _intercept_unary_response(self, behavior, request_or_iterator, context, request_streaming, full_method): + with self._set_remote_context(context): + attributes = self._create_attributes(context, full_method) + + with self._tracer.start_as_current_span( + name=full_method, + kind=trace.SpanKind.SERVER, + attributes=attributes, + end_on_exit=True, + record_exception=False, + set_status_on_exception=False + ) as span: + + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # wrap / log the request (iterator) + if request_streaming: + request_or_iterator = self._log_streaming_request( + request_or_iterator, span, attributes + ) + else: + self._log_unary_request( + request_or_iterator, span, attributes + ) + + # call the actual RPC and track the duration + with self._record_duration(attributes, context): + response_or_iterator = behavior(request_or_iterator, context) + + # log the response (iterator) + self._log_unary_response( + response_or_iterator, span, attributes, context + ) + + return response_or_iterator + + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + + def _intercept_streaming_response(self, behavior, request_or_iterator, context, request_streaming, full_method): + with self._set_remote_context(context): + attributes = self._create_attributes(context, full_method) + + with self._tracer.start_as_current_span( + name=full_method, + kind=trace.SpanKind.SERVER, + attributes=attributes, + end_on_exit=True, + record_exception=False, + set_status_on_exception=False + ) as span: + + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # wrap / log the request (iterator) + if request_streaming: + request_or_iterator = self._log_streaming_request( + request_or_iterator, span, attributes + ) + else: + self._log_unary_request( + request_or_iterator, span, attributes + ) + + # call the actual RPC and track the duration + with self._record_duration(attributes, context): + response_or_iterator = behavior(request_or_iterator, context) + + # log the response (iterator) + yield from self._log_streaming_response( + response_or_iterator, span, attributes, context + ) + + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + def _log_unary_request(self, request, active_span, attributes): message_size_by = request.ByteSize() _add_message_event( @@ -429,7 +487,7 @@ def _log_unary_response(self, response, active_span, attributes, context): self._response_size_histogram.record(message_size_by, attributes) self._responses_per_rpc_histogram.record(1, attributes) - def _log_stream_requests(self, request_iterator, active_span, attributes): + def _log_streaming_request(self, request_iterator, active_span, attributes): req_id = 1 for req_id, msg in enumerate(request_iterator, start=1): message_size_by = msg.ByteSize() @@ -441,44 +499,20 @@ def _log_stream_requests(self, request_iterator, active_span, attributes): self._requests_per_rpc_histogram.record(req_id, attributes) - def _log_stream_responses(self, response_iterator, active_span, attributes, context): - with trace.use_span( - active_span, - end_on_exit=True, - record_exception=False, - set_status_on_exception=False - ): - try: - res_id = 1 - for res_id, msg in enumerate(response_iterator, start=1): - message_size_by = msg.ByteSize() - _add_message_event( - active_span, MessageTypeValues.SENT.value, message_size_by, message_id=res_id - ) - self._response_size_histogram.record(message_size_by, attributes) - yield msg - except Exception as exc: - # Bare exceptions are likely to be gRPC aborts, which - # we handle in our context wrapper. - # Here, we're interested in uncaught exceptions. - # pylint:disable=unidiomatic-typecheck - if type(exc) != Exception: - active_span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, - grpc.StatusCode.UNKNOWN.value[0] - ) - active_span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{type(exc).__name__}: {exc}", - ) - ) - active_span.record_exception(exc) - raise exc - finally: - if context._code != grpc.StatusCode.OK: - attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] - self._responses_per_rpc_histogram.record(res_id, attributes) + def _log_streaming_response(self, response_iterator, active_span, attributes, context): + try: + res_id = 1 + for res_id, msg in enumerate(response_iterator, start=1): + message_size_by = msg.ByteSize() + _add_message_event( + active_span, MessageTypeValues.SENT.value, message_size_by, message_id=res_id + ) + self._response_size_histogram.record(message_size_by, attributes) + yield msg + finally: + if context._code != grpc.StatusCode.OK: + attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] + self._responses_per_rpc_histogram.record(res_id, attributes) @contextmanager def _record_duration(self, attributes, context): @@ -486,7 +520,7 @@ def _record_duration(self, attributes, context): try: yield finally: - duration = max(round((_time_ns() - start) * 1000), 0) + duration = max(round((_time_ns() - start) / 1000), 0) if context._code != grpc.StatusCode.OK: attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] self._duration_histogram.record(duration, attributes) From fa356f2825ba2fdf42701bc96dd50ffbe663b4c4 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Thu, 28 Jul 2022 19:11:00 +0200 Subject: [PATCH 07/17] Added types for typing --- .../instrumentation/grpc/_types.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py new file mode 100644 index 0000000000..81f12ba943 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py @@ -0,0 +1,29 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Internal types.""" + +from typing import Dict, Iterator, TypeVar, Union + +import grpc + + +Metadata = Dict[str, Union[str, bytes]] +"""Metadata type.""" + +ProtoMessage = TypeVar("ProtoMessage") +"""Protobuf message type.""" + +ProtoMessageOrIterator = Union[ProtoMessage, Iterator[ProtoMessage]] +"""Type for a protobuf message or a iterator of protobuf messages.""" From 5a33def2153dfc3d7c05e8ee69f1f0a4b5f2e751 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Thu, 28 Jul 2022 19:11:31 +0200 Subject: [PATCH 08/17] added event metric recorder --- .../instrumentation/grpc/_utilities.py | 391 +++++++++++++++++- 1 file changed, 375 insertions(+), 16 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py index b6ff7d311a..543254c396 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py @@ -14,20 +14,379 @@ """Internal utilities.""" +from collections import namedtuple +from contextlib import contextmanager +from enum import Enum +from timeit import default_timer +from typing import Generator, Iterator -class RpcInfo: - def __init__( - self, - full_method=None, - metadata=None, - timeout=None, - request=None, - response=None, - error=None, - ): - self.full_method = full_method - self.metadata = metadata - self.timeout = timeout - self.request = request - self.response = response - self.error = error +import grpc +from opentelemetry.instrumentation.grpc._types import ProtoMessage +from opentelemetry.metrics import Meter +from opentelemetry.trace import Span +from opentelemetry.semconv.trace import MessageTypeValues, SpanAttributes +from opentelemetry.util.types import Attributes + + +_MESSAGE: str = "message" +"""Event name of a message.""" + + +def _add_message_event( + active_span: Span, + message_type: str, + message_size_by: int, + message_id: int = 1 +) -> None: + """Adds a message event of an RPC to an active span. + + Args: + active_span (Span): The active span in which to record the message + as event. + message_type (str): The message type value as str, either "SENT" or + "RECEIVED". + message_size_by (int): The (uncompressed) message size in bytes as int. + message_id (int, optional): The message ID. Defaults to 1. + """ + + active_span.add_event( + _MESSAGE, + { + SpanAttributes.MESSAGE_TYPE: message_type, + SpanAttributes.MESSAGE_ID: message_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: message_size_by, + } + ) + + +class _ClientCallDetails( + namedtuple( + "_ClientCallDetails", + ("method", "timeout", "metadata", "credentials", + "wait_for_ready", "compression") + ), + grpc.ClientCallDetails +): + pass + + +class _MetricKind(Enum): + """Specifies the kind of the metric. + """ + + #: Indicates that the metric is of a server. + CLIENT = "client" + + #: Indicates that the metric is of a server. + SERVER = "server" + + +class _EventMetricRecorder: + """Internal class for recording messages as event and in the histograms + and for recording the duration of a RPC. + """ + + def __init__(self, meter: Meter, kind: _MetricKind) -> None: + """Initializes the _EventMetricRecorder. + + Args: + meter (Meter): The meter to create the metrics. + kind (str): The kind of the metric recorder, either for a "client" + or "server". + """ + + self._meter = meter + + metric_kind = _MetricKind(kind) + self._duration_histogram = self._meter.create_histogram( + name=f"rpc.{metric_kind.value}.duration", + unit="ms", + description="Measures duration of RPC", + ) + self._request_size_histogram = self._meter.create_histogram( + name=f"rpc.{metric_kind.value}.request.size", + unit="By", + description="Measures size of RPC request messages (uncompressed)", + ) + self._response_size_histogram = self._meter.create_histogram( + name=f"rpc.{metric_kind.value}.response.size", + unit="By", + description="Measures size of RPC response messages " + "(uncompressed)", + ) + self._requests_per_rpc_histogram = self._meter.create_histogram( + name=f"rpc.{metric_kind.value}.requests_per_rpc", + unit="1", + description="Measures the number of messages received per RPC. " + "Should be 1 for all non-streaming RPCs", + ) + self._responses_per_rpc_histogram = self._meter.create_histogram( + name=f"rpc.{metric_kind.value}.responses_per_rpc", + unit="1", + description="Measures the number of messages sent per RPC. " + "Should be 1 for all non-streaming RPCs", + ) + + def _record_unary_request( + self, + active_span: Span, + request: ProtoMessage, + message_type: MessageTypeValues, + metric_attributes: Attributes + ) -> None: + """Records a unary request. + + The request is recorded as event, its size in the request-size- + histogram and a one for a unary request in the requests-per-RPC- + histogram. + + Args: + active_span (Span): The active span in which to record the request + as event. + request (ProtoMessage): The request message. + message_type (MessageTypeValues): The message type value. + metric_attributes (Attributes): The attributes to record in the + metrics. + """ + + message_size_by = request.ByteSize() + _add_message_event(active_span, message_type.value, message_size_by) + self._request_size_histogram.record(message_size_by, metric_attributes) + self._requests_per_rpc_histogram.record(1, metric_attributes) + + def _record_response( + self, + active_span: Span, + response: ProtoMessage, + message_type: MessageTypeValues, + metric_attributes: Attributes, + response_id: int = 1 + ) -> None: + """Records a unary OR streaming response. + + The response is recorded as event and its size in the response-size- + histogram. + + Args: + active_span (Span): The active span in which to record the response + as event. + response (ProtoMessage): The response message. + message_type (MessageTypeValues): The message type value. + metric_attributes (Attributes): The attributes to record in the + metrics. + response_id (int, optional): The response ID. Defaults to 1. + """ + + message_size_by = response.ByteSize() + _add_message_event( + active_span, + message_type.value, + message_size_by, + message_id=response_id + ) + self._response_size_histogram.record( + message_size_by, metric_attributes + ) + + def _record_responses_per_rpc( + self, + responses_per_rpc: int, + metric_attributes: Attributes + ) -> None: + """Records the number of responses in the responses-per-RPC-histogram + for a streaming response. + + + Args: + responses_per_rpc (int): The number of responses. + metric_attributes (Attributes): The attributes to record in the + metric. + """ + + self._responses_per_rpc_histogram.record( + responses_per_rpc, metric_attributes + ) + + def _record_unary_response( + self, + active_span: Span, + response: ProtoMessage, + message_type: MessageTypeValues, + metric_attributes: Attributes + ) -> None: + """Records a unary response. + + The response is recorded as event, its size in the response-size- + histogram and a one for a unary response in the responses-per-RPC- + histogram. + + Args: + active_span (Span): The active span in which to record the response + as event. + response (ProtoMessage): The response message. + message_type (MessageTypeValues): The message type value. + metric_attributes (Attributes): The attributes to record in the + metrics. + """ + + self._record_response( + active_span, response, message_type, metric_attributes + ) + self._record_responses_per_rpc(1, metric_attributes) + + def _record_streaming_request( + self, + active_span: Span, + request_iterator: Iterator[ProtoMessage], + message_type: MessageTypeValues, + metric_attributes: Attributes + ) -> Iterator[ProtoMessage]: + """Records a streaming request. + + The requests are recorded as events, their size in the request-size- + histogram and the total number of requests in the requests-per-RPC- + histogram. + + Args: + active_span (Span): The active span in which to record the request + as event. + request_iterator (Iterator[ProtoMessage]): The iterator over the + request messages. + message_type (MessageTypeValues): The message type value. + metric_attributes (Attributes): The attributes to record in the + metrics. + + Yields: + Iterator[ProtoMessage]: The iterator over the recorded request + messages. + """ + + try: + req_id = 0 + for req_id, request in enumerate(request_iterator, start=1): + message_size_by = request.ByteSize() + _add_message_event( + active_span, + message_type.value, + message_size_by, + message_id=req_id + ) + self._request_size_histogram.record( + message_size_by, metric_attributes + ) + yield request + finally: + self._requests_per_rpc_histogram.record(req_id, metric_attributes) + + def _record_streaming_response( + self, + active_span: Span, + response_iterator: Iterator[ProtoMessage], + message_type: MessageTypeValues, + metric_attributes: Attributes + ) -> Iterator[ProtoMessage]: + """Records a streaming response. + + The responses are recorded as events, their size in the response-size- + histogram and the total number of responses in the responses-per-RPC- + histogram. + + Args: + active_span (Span): The active span in which to record the response + as event. + response_iterator (Iterator[ProtoMessage]): The iterator over the + response messages. + message_type (MessageTypeValues): The message type value. + metric_attributes (Attributes): The attributes to record in the + metrics. + + Yields: + Iterator[ProtoMessage]: The iterator over the recorded response + messages. + """ + + try: + res_id = 0 + for res_id, response in enumerate(response_iterator, start=1): + self._record_response( + active_span, + response, + message_type, + metric_attributes, + response_id=res_id + ) + yield response + finally: + self._record_responses_per_rpc(res_id, metric_attributes) + + def _start_duration_measurement(self) -> float: + """Starts a duration measurement and returns the start time. + + Returns: + float: The start time. + """ + + return default_timer() + + def _record_duration( + self, + start_time: float, + metric_attributes: Attributes, + context: grpc.RpcContext + ) -> None: + """Records a duration of an RPC in the duration histogram. The duration + is calculated as difference between the call of this method and the + start time which has been returned by _start_duration_measurement.d + + Args: + start_time (float): The start time. + metric_attributes (Attributes): The attributes to record in the + metric. + context (grpc.RpcContext): The RPC context to update the status + code of the RPC. + """ + + duration = max(round((default_timer() - start_time) * 1000), 0) + if context.code() in (None, grpc.StatusCode.OK): + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) + else: + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + context.code().value[0] + ) + self._duration_histogram.record(duration, metric_attributes) + + @contextmanager + def _record_duration_manager( + self, + metric_attributes: Attributes, + context: grpc.RpcContext + ) -> Generator[None, None, None]: + """Returns a context manager to measure the duration of an RPC in the + duration histogram. + + Args: + metric_attributes (Attributes): The attributes to record in the + metric. + context (grpc.RpcContext): The RPC context to update the status + code of the RPC. + + Yields: + Generator[None, None, None]: The context manager. + """ + + start_time = default_timer() + try: + yield + finally: + duration = max(round((default_timer() - start_time) * 1000), 0) + if context.code() in (None, grpc.StatusCode.OK): + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) + else: + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + context.code().value[0] + ) + self._duration_histogram.record(duration, metric_attributes) From 65b15376721ea648784f0683b0d38a33322e8599 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Thu, 28 Jul 2022 19:11:43 +0200 Subject: [PATCH 09/17] Added typing, seperated calls by unary/stream, distinguished span and metric attributes --- .../instrumentation/grpc/_server.py | 625 +++++++++++------- 1 file changed, 374 insertions(+), 251 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index 4f9c23659d..c070b99232 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -21,32 +21,39 @@ Implementation of the service-side open-telemetry interceptor. """ +import copy import logging from contextlib import contextmanager +from typing import Callable, Dict, Iterable, Iterator, Generator, NoReturn, Optional import grpc -from opentelemetry import trace +from opentelemetry import metrics, trace from opentelemetry.context import attach, detach +from opentelemetry.instrumentation.grpc._types import Metadata, ProtoMessage, ProtoMessageOrIterator +from opentelemetry.instrumentation.grpc._utilities import _EventMetricRecorder, _MetricKind from opentelemetry.propagate import extract from opentelemetry.semconv.trace import MessageTypeValues, RpcSystemValues, SpanAttributes from opentelemetry.trace.status import Status, StatusCode -from opentelemetry.util._time import _time_ns +from opentelemetry.util.types import Attributes logger = logging.getLogger(__name__) -_MESSAGE = "message" -"""event name of a message.""" - _RPC_USER_AGENT = "rpc.user_agent" """span attribute for RPC user agent.""" # wrap an RPC call # see https://github.com/grpc/grpc/issues/18191 -def _wrap_rpc_behavior(handler, continuation): +def _wrap_rpc_behavior( + handler: Optional[grpc.RpcMethodHandler], + continuation: Callable[ + [ProtoMessageOrIterator, grpc.ServicerContext], + ProtoMessageOrIterator + ] +) -> Optional[grpc.RpcMethodHandler]: if handler is None: return None @@ -72,26 +79,14 @@ def _wrap_rpc_behavior(handler, continuation): ) -def _add_message_event( - span, - message_type, - message_size_by, - message_id=1 -): - span.add_event( - _MESSAGE, - { - SpanAttributes.MESSAGE_TYPE: message_type, - SpanAttributes.MESSAGE_ID: message_id, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: message_size_by, - } - ) - - # pylint:disable=abstract-method class _OpenTelemetryServicerContext(grpc.ServicerContext): - def __init__(self, servicer_context, active_span): + def __init__( + self, + servicer_context: grpc.ServicerContext, + active_span: trace.Span + ) -> None: self._servicer_context = servicer_context self._active_span = active_span self._code = grpc.StatusCode.OK @@ -101,13 +96,13 @@ def __init__(self, servicer_context, active_span): def __getattr__(self, attr): return getattr(self._servicer_context, attr) - def is_active(self, *args, **kwargs): - return self._servicer_context.is_active(*args, **kwargs) + # Interface of grpc.RpcContext - def time_remaining(self, *args, **kwargs): - return self._servicer_context.time_remaining(*args, **kwargs) + # pylint: disable=invalid-name + def add_callback(self, fn: Callable[[], None]) -> None: + return self._servicer_context.add_callback(fn) - def cancel(self, *args, **kwargs): + def cancel(self) -> None: self._code = grpc.StatusCode.CANCELLED self._details = grpc.StatusCode.CANCELLED.value[1] self._active_span.set_attribute( @@ -119,42 +114,17 @@ def cancel(self, *args, **kwargs): description=f"{self._code}: {self._details}", ) ) - return self._servicer_context.cancel(*args, **kwargs) - - def add_callback(self, *args, **kwargs): - return self._servicer_context.add_callback(*args, **kwargs) - - def disable_next_message_compression(self): - return self._service_context.disable_next_message_compression() - - def invocation_metadata(self): - return self._servicer_context.invocation_metadata() - - def peer(self): - return self._servicer_context.peer() - - def peer_identities(self): - return self._servicer_context.peer_identities() + return self._servicer_context.cancel() - def peer_identity_key(self): - return self._servicer_context.peer_identity_key() - - def auth_context(self): - return self._servicer_context.auth_context() - - def set_compression(self, compression): - return self._servicer_context.set_compression(compression) - - def send_initial_metadata(self, *args, **kwargs): - return self._servicer_context.send_initial_metadata(*args, **kwargs) + def is_active(self) -> bool: + return self._servicer_context.is_active() - def set_trailing_metadata(self, *args, **kwargs): - return self._servicer_context.set_trailing_metadata(*args, **kwargs) + def time_remaining(self) -> Optional[float]: + return self._servicer_context.time_remaining() - def trailing_metadata(self): - return self._servicer_context.trailing_metadata() + # Interface of grpc.ServicerContext - def abort(self, code, details): + def abort(self, code: grpc.StatusCode, details: str) -> NoReturn: if not hasattr(self._servicer_context, "abort"): raise RuntimeError( "abort() is not supported with the installed version of grpcio" @@ -172,21 +142,51 @@ def abort(self, code, details): ) return self._servicer_context.abort(code, details) - def abort_with_status(self, status): + def abort_with_status(self, status: grpc.Status) -> NoReturn: if not hasattr(self._servicer_context, "abort_with_status"): raise RuntimeError( - "abort_with_status() is not supported with the installed version of grpcio" + "abort_with_status() is not supported with the installed " + "version of grpcio" ) return self._servicer_context.abort_with_status(status) - def code(self): + def auth_context(self) -> Dict[str, Iterable[bytes]]: + return self._servicer_context.auth_context() + + def code(self) -> grpc.StatusCode: if not hasattr(self._servicer_context, "code"): raise RuntimeError( "code() is not supported with the installed version of grpcio" ) return self._servicer_context.code() - def set_code(self, code): + def details(self) -> str: + if not hasattr(self._servicer_context, "details"): + raise RuntimeError( + "details() is not supported with the installed version of " + "grpcio" + ) + return self._servicer_context.details() + + def disable_next_message_compression(self) -> None: + return self._service_context.disable_next_message_compression() + + def invocation_metadata(self) -> Metadata: + return self._servicer_context.invocation_metadata() + + def peer(self) -> str: + return self._servicer_context.peer() + + def peer_identities(self) -> Optional[Iterable[bytes]]: + return self._servicer_context.peer_identities() + + def peer_identity_key(self) -> Optional[str]: + return self._servicer_context.peer_identity_key() + + def send_initial_metadata(self, initial_metadata: Metadata) -> None: + return self._servicer_context.send_initial_metadata(initial_metadata) + + def set_code(self, code: grpc.StatusCode) -> None: self._code = code # use details if we already have it, otherwise the status description details = self._details or code.value[1] @@ -202,15 +202,10 @@ def set_code(self, code): ) return self._servicer_context.set_code(code) - def details(self): - if not hasattr(self._servicer_context, "details"): - raise RuntimeError( - "details() is not supported with the installed version of " - "grpcio" - ) - return self._servicer_context.details() + def set_compression(self, compression: grpc.Compression) -> None: + return self._servicer_context.set_compression(compression) - def set_details(self, details): + def set_details(self, details: str) -> None: self._details = details if self._code != grpc.StatusCode.OK: self._active_span.set_status( @@ -221,11 +216,20 @@ def set_details(self, details): ) return self._servicer_context.set_details(details) + def set_trailing_metadata(self, trailing_metadata: Metadata) -> None: + return self._servicer_context.set_trailing_metadata(trailing_metadata) + + def trailing_metadata(self) -> Metadata: + return self._servicer_context.trailing_metadata() + # pylint:disable=abstract-method # pylint:disable=no-self-use # pylint:disable=unused-argument -class OpenTelemetryServerInterceptor(grpc.ServerInterceptor): +class OpenTelemetryServerInterceptor( + _EventMetricRecorder, + grpc.ServerInterceptor +): """ A gRPC server interceptor, to add OpenTelemetry. @@ -245,38 +249,19 @@ class OpenTelemetryServerInterceptor(grpc.ServerInterceptor): """ - def __init__(self, meter, tracer): - self._meter = meter + def __init__( + self, + meter: metrics.Meter, + tracer: trace.Tracer + ) -> None: + super().__init__(meter, _MetricKind.SERVER) self._tracer = tracer - self._duration_histogram = self._meter.create_histogram( - name="rpc.server.duration", - unit="ms", - description="measures duration of inbound RPC", - ) - self._request_size_histogram = self._meter.create_histogram( - name="rpc.server.request.size", - unit="By", - description="measures size of RPC request messages (uncompressed)", - ) - self._response_size_histogram = self._meter.create_histogram( - name="rpc.server.response.size", - unit="By", - description="measures size of RPC response messages (uncompressed)", - ) - self._requests_per_rpc_histogram = self._meter.create_histogram( - name="rpc.server.requests_per_rpc", - unit="requests", - description="measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs", - ) - self._responses_per_rpc_histogram = self._meter.create_histogram( - name="rpc.server.responses_per_rpc", - unit="responses", - description="measures the number of messages sent per RPC. Should be 1 for all non-streaming RPCs", - ) - @contextmanager - def _set_remote_context(self, context): + def _set_remote_context( + self, + context: grpc.ServicerContext + ) -> Generator[None, None, None]: metadata = context.invocation_metadata() if metadata: md_dict = {md.key: md.value for md in metadata} @@ -289,22 +274,19 @@ def _set_remote_context(self, context): else: yield - def _create_attributes(self, context, full_method): + def _create_attributes( + self, + context: grpc.ServicerContext, + full_method: str + ) -> Attributes: # standard attributes + service, method = full_method.lstrip("/").split("/", 1) attributes = { SpanAttributes.RPC_SYSTEM: RpcSystemValues.GRPC.value, - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], + SpanAttributes.RPC_SERVICE: service, + SpanAttributes.RPC_METHOD: method } - # add service and method attributes - service, method = full_method.lstrip("/").split("/", 1) - attributes.update( - { - SpanAttributes.RPC_SERVICE: service, - SpanAttributes.RPC_METHOD: method, - } - ) - # add some attributes from the metadata metadata = dict(context.invocation_metadata()) if "user-agent" in metadata: @@ -336,18 +318,48 @@ def _create_attributes(self, context, full_method): return attributes - def intercept_service(self, continuation, handler_call_details): - def telemetry_wrapper(behavior, request_streaming, response_streaming): - def telemetry_interceptor(request_or_iterator, context): - if response_streaming: - return self._intercept_streaming_response( + def intercept_service( + self, + continuation: Callable[ + [grpc.HandlerCallDetails], Optional[grpc.RpcMethodHandler] + ], + handler_call_details: grpc.HandlerCallDetails + ) -> Optional[grpc.RpcMethodHandler]: + + def telemetry_wrapper( + behavior: Callable[ + [ProtoMessageOrIterator, grpc.ServicerContext], + ProtoMessageOrIterator + ], + request_streaming: bool, + response_streaming: bool + ) -> Callable[ + [ProtoMessageOrIterator, grpc.ServicerContext], + ProtoMessageOrIterator + ]: + + def telemetry_interceptor( + request_or_iterator: ProtoMessageOrIterator, + context: grpc.ServicerContext + ) -> ProtoMessageOrIterator: + if request_streaming and response_streaming: + return self.intercept_stream_stream( behavior, request_or_iterator, context, - request_streaming, handler_call_details.method + handler_call_details.method ) - - return self._intercept_unary_response( + if not request_streaming and response_streaming: + return self.intercept_unary_stream( + behavior, request_or_iterator, context, + handler_call_details.method + ) + if request_streaming and not response_streaming: + return self.intercept_stream_unary( + behavior, request_or_iterator, context, + handler_call_details.method + ) + return self.intercept_unary_unary( behavior, request_or_iterator, context, - request_streaming, handler_call_details.method + handler_call_details.method ) return telemetry_interceptor @@ -357,170 +369,281 @@ def telemetry_interceptor(request_or_iterator, context): telemetry_wrapper ) - def _intercept_unary_response(self, behavior, request_or_iterator, context, request_streaming, full_method): + def intercept_unary_unary( + self, + continuation: Callable[ + [ProtoMessage, grpc.ServicerContext], ProtoMessage + ], + request: ProtoMessage, + context: grpc.ServicerContext, + full_method: str + ) -> ProtoMessage: with self._set_remote_context(context): - attributes = self._create_attributes(context, full_method) + metric_attributes = self._create_attributes(context, full_method) + span_attributes = copy.deepcopy(metric_attributes) + span_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) with self._tracer.start_as_current_span( name=full_method, kind=trace.SpanKind.SERVER, - attributes=attributes, + attributes=span_attributes, end_on_exit=True, record_exception=False, set_status_on_exception=False ) as span: + with self._record_duration_manager(metric_attributes, context): + + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # record the request + self._record_unary_request( + span, + request, + MessageTypeValues.RECEIVED, + metric_attributes + ) - try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) + # call the actual RPC + response = continuation(request, context) - # wrap / log the request (iterator) - if request_streaming: - request_or_iterator = self._log_streaming_request( - request_or_iterator, span, attributes - ) - else: - self._log_unary_request( - request_or_iterator, span, attributes + # record the response + self._record_unary_response( + span, + response, + MessageTypeValues.SENT, + metric_attributes ) - # call the actual RPC and track the duration - with self._record_duration(attributes, context): - response_or_iterator = behavior(request_or_iterator, context) + return response + + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + + def intercept_unary_stream( + self, + continuation: Callable[ + [ProtoMessage, grpc.ServicerContext], Iterator[ProtoMessage] + ], + request: ProtoMessage, + context: grpc.ServicerContext, + full_method: str + ) -> Iterator[ProtoMessage]: + with self._set_remote_context(context): + metric_attributes = self._create_attributes(context, full_method) + span_attributes = copy.deepcopy(metric_attributes) + span_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) - # log the response (iterator) - self._log_unary_response( - response_or_iterator, span, attributes, context - ) + with self._tracer.start_as_current_span( + name=full_method, + kind=trace.SpanKind.SERVER, + attributes=span_attributes, + end_on_exit=True, + record_exception=False, + set_status_on_exception=False + ) as span: - return response_or_iterator - - except Exception as exc: - # Bare exceptions are likely to be gRPC aborts, which - # we handle in our context wrapper. - # Here, we're interested in uncaught exceptions. - # pylint:disable=unidiomatic-typecheck - if type(exc) != Exception: - span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, - grpc.StatusCode.UNKNOWN.value[0] + with self._record_duration_manager(metric_attributes, context): + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # record the request + self._record_unary_request( + span, + request, + MessageTypeValues.RECEIVED, + metric_attributes ) - span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{type(exc).__name__}: {exc}", - ) + + # call the actual RPC + response_iterator = continuation(request, context) + + # wrap the response iterator with a recorder + yield from self._record_streaming_response( + span, + response_iterator, + MessageTypeValues.SENT, + metric_attributes ) - span.record_exception(exc) - raise exc - def _intercept_streaming_response(self, behavior, request_or_iterator, context, request_streaming, full_method): + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + + def intercept_stream_unary( + self, + continuation: Callable[ + [Iterator[ProtoMessage], grpc.ServicerContext], ProtoMessage + ], + request_iterator: Iterator[ProtoMessage], + context: grpc.ServicerContext, + full_method: str + ) -> ProtoMessage: with self._set_remote_context(context): - attributes = self._create_attributes(context, full_method) + metric_attributes = self._create_attributes(context, full_method) + span_attributes = copy.deepcopy(metric_attributes) + span_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) with self._tracer.start_as_current_span( name=full_method, kind=trace.SpanKind.SERVER, - attributes=attributes, + attributes=span_attributes, end_on_exit=True, record_exception=False, set_status_on_exception=False ) as span: - - try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - - # wrap / log the request (iterator) - if request_streaming: - request_or_iterator = self._log_streaming_request( - request_or_iterator, span, attributes - ) - else: - self._log_unary_request( - request_or_iterator, span, attributes + with self._record_duration_manager(metric_attributes, context): + + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # wrap the request iterator with a recorder + request_iterator = self._record_streaming_request( + span, + request_iterator, + MessageTypeValues.RECEIVED, + metric_attributes ) - # call the actual RPC and track the duration - with self._record_duration(attributes, context): - response_or_iterator = behavior(request_or_iterator, context) + # call the actual RPC + response = continuation(request_iterator, context) - # log the response (iterator) - yield from self._log_streaming_response( - response_or_iterator, span, attributes, context + # record the response + self._record_unary_response( + span, + response, + MessageTypeValues.SENT, + metric_attributes ) - except Exception as exc: - # Bare exceptions are likely to be gRPC aborts, which - # we handle in our context wrapper. - # Here, we're interested in uncaught exceptions. - # pylint:disable=unidiomatic-typecheck - if type(exc) != Exception: - span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, - grpc.StatusCode.UNKNOWN.value[0] - ) - span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{type(exc).__name__}: {exc}", + return response + + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] ) - ) - span.record_exception(exc) - raise exc + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc + + def intercept_stream_stream( + self, + continuation: Callable[ + [Iterator[ProtoMessage], grpc.ServicerContext], + Iterator[ProtoMessage] + ], + request_iterator: Iterator[ProtoMessage], + context: grpc.ServicerContext, + full_method: str + ) -> Iterator[ProtoMessage]: + with self._set_remote_context(context): + metric_attributes = self._create_attributes(context, full_method) + span_attributes = copy.deepcopy(metric_attributes) + span_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + grpc.StatusCode.OK.value[0] + ) - def _log_unary_request(self, request, active_span, attributes): - message_size_by = request.ByteSize() - _add_message_event( - active_span, MessageTypeValues.RECEIVED.value, message_size_by - ) - self._request_size_histogram.record(message_size_by, attributes) - self._requests_per_rpc_histogram.record(1, attributes) + with self._tracer.start_as_current_span( + name=full_method, + kind=trace.SpanKind.SERVER, + attributes=span_attributes, + end_on_exit=True, + record_exception=False, + set_status_on_exception=False + ) as span: - def _log_unary_response(self, response, active_span, attributes, context): - message_size_by = response.ByteSize() - _add_message_event( - active_span, MessageTypeValues.SENT.value, message_size_by - ) - if context._code != grpc.StatusCode.OK: - attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] - self._response_size_histogram.record(message_size_by, attributes) - self._responses_per_rpc_histogram.record(1, attributes) - - def _log_streaming_request(self, request_iterator, active_span, attributes): - req_id = 1 - for req_id, msg in enumerate(request_iterator, start=1): - message_size_by = msg.ByteSize() - _add_message_event( - active_span, MessageTypeValues.RECEIVED.value, message_size_by, message_id=req_id - ) - self._request_size_histogram.record(message_size_by, attributes) - yield msg + with self._record_duration_manager(metric_attributes, context): + try: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + + # wrap the request iterator with a recorder + request_iterator = self._record_streaming_request( + span, + request_iterator, + MessageTypeValues.RECEIVED, + metric_attributes + ) - self._requests_per_rpc_histogram.record(req_id, attributes) + # call the actual RPC + response_iterator = continuation( + request_iterator, context + ) - def _log_streaming_response(self, response_iterator, active_span, attributes, context): - try: - res_id = 1 - for res_id, msg in enumerate(response_iterator, start=1): - message_size_by = msg.ByteSize() - _add_message_event( - active_span, MessageTypeValues.SENT.value, message_size_by, message_id=res_id - ) - self._response_size_histogram.record(message_size_by, attributes) - yield msg - finally: - if context._code != grpc.StatusCode.OK: - attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] - self._responses_per_rpc_histogram.record(res_id, attributes) + # wrap the response iterator with a recorder + yield from self._record_streaming_response( + span, + response_iterator, + MessageTypeValues.SENT, + metric_attributes + ) - @contextmanager - def _record_duration(self, attributes, context): - start = _time_ns() - try: - yield - finally: - duration = max(round((_time_ns() - start) / 1000), 0) - if context._code != grpc.StatusCode.OK: - attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = context._code.value[0] - self._duration_histogram.record(duration, attributes) + except Exception as exc: + # Bare exceptions are likely to be gRPC aborts, which + # we handle in our context wrapper. + # Here, we're interested in uncaught exceptions. + # pylint:disable=unidiomatic-typecheck + if type(exc) != Exception: + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, + grpc.StatusCode.UNKNOWN.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{type(exc).__name__}: {exc}", + ) + ) + span.record_exception(exc) + raise exc From 6a831e2e555f475ce0f07e0070b9c0fb8e796ab3 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Thu, 28 Jul 2022 19:12:00 +0200 Subject: [PATCH 10/17] added tests of metrics --- .../tests/_server.py | 90 +- .../tests/test_server_interceptor.py | 1175 +++++++++++++---- 2 files changed, 987 insertions(+), 278 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py index 5375253c01..f46cce722f 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py @@ -13,6 +13,7 @@ # limitations under the License. from concurrent import futures +import time import grpc @@ -26,51 +27,98 @@ class TestServer(test_server_pb2_grpc.GRPCTestServerServicer): # pylint: disable=no-self-use def SimpleMethod(self, request, context): - if request.request_data == "error": + if request.request_data == "abort": + context.abort( + grpc.StatusCode.FAILED_PRECONDITION, request.request_data + ) + elif request.request_data == "cancel": + context.cancel() + return test_server_pb2.Response() + elif request.request_data == "error": context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(request.request_data) return test_server_pb2.Response() - response = test_server_pb2.Response( + elif request.request_data == "exception": + raise ValueError(request.request_data) + elif request.request_data == "sleep": + time.sleep(0.5) + + return test_server_pb2.Response( server_id=SERVER_ID, response_data="data" ) - return response def ClientStreamingMethod(self, request_iterator, context): data = list(request_iterator) - if data[0].request_data == "error": + + if data[0].request_data == "abort": + context.abort( + grpc.StatusCode.FAILED_PRECONDITION, data[0].request_data + ) + elif data[0].request_data == "cancel": + context.cancel() + return test_server_pb2.Response() + elif data[0].request_data == "error": context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(data[0].request_data) return test_server_pb2.Response() - response = test_server_pb2.Response( + elif data[0].request_data == "exception": + raise ValueError(data[0].request_data) + elif data[0].request_data == "sleep": + time.sleep(0.5) + + return test_server_pb2.Response( server_id=SERVER_ID, response_data="data" ) - return response def ServerStreamingMethod(self, request, context): - if request.request_data == "error": + yield test_server_pb2.Response( + server_id=SERVER_ID, response_data="data" + ) + + if request.request_data == "abort": context.abort( - code=grpc.StatusCode.INVALID_ARGUMENT, - details="server stream error", + grpc.StatusCode.FAILED_PRECONDITION, request.request_data ) + elif request.request_data == "cancel": + context.cancel() return test_server_pb2.Response() + elif request.request_data == "error": + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(request.request_data) + return test_server_pb2.Response() + elif request.request_data == "exception": + raise ValueError(request.request_data) + elif request.request_data == "sleep": + time.sleep(0.5) - # create a generator - def response_messages(): - for _ in range(5): - response = test_server_pb2.Response( - server_id=SERVER_ID, response_data="data" - ) - yield response - - return response_messages() + for _ in range(5): + yield test_server_pb2.Response( + server_id=SERVER_ID, response_data="data" + ) def BidirectionalStreamingMethod(self, request_iterator, context): data = list(request_iterator) - if data[0].request_data == "error": + + yield test_server_pb2.Response( + server_id=SERVER_ID, response_data="data" + ) + + if data[0].request_data == "abort": context.abort( - code=grpc.StatusCode.INVALID_ARGUMENT, - details="bidirectional error", + grpc.StatusCode.FAILED_PRECONDITION, data[0].request_data ) + elif data[0].request_data == "cancel": + context.cancel() + return + elif data[0].request_data == "error": + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(data[0].request_data) return + elif data[0].request_data == "exception": + raise ValueError(data[0].request_data) + elif data[0].request_data == "sleep": + time.sleep(0.5) for _ in range(5): yield test_server_pb2.Response( diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py index b0669032ca..2d06cb92fd 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py @@ -15,18 +15,17 @@ # pylint:disable=unused-argument # pylint:disable=no-self-use +from ftplib import error_perm +import logging import threading from concurrent import futures import grpc +from opentelemetry import trace, metrics import opentelemetry.instrumentation.grpc -from opentelemetry import trace -from opentelemetry.instrumentation.grpc import ( - GrpcInstrumentorServer, - server_interceptor, -) -from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer, server_interceptor +from opentelemetry.sdk.metrics.export import Histogram, HistogramDataPoint from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.test.test_base import TestBase from opentelemetry.trace import StatusCode @@ -36,25 +35,38 @@ GRPCTestServerServicer, add_GRPCTestServerServicer_to_server, ) - - -class Servicer(GRPCTestServerServicer): - """Our test servicer""" - - # pylint:disable=C0103 - def SimpleMethod(self, request, context): - return Response( - server_id=request.client_id, - response_data=request.request_data, - ) - - # pylint:disable=C0103 - def ServerStreamingMethod(self, request, context): - for data in ("one", "two", "three"): - yield Response( - server_id=request.client_id, - response_data=data, - ) +from ._server import TestServer + + +_expected_metric_names = { + "rpc.server.duration": ( + Histogram, + "ms", + "Measures duration of RPC" + ), + "rpc.server.request.size": ( + Histogram, + "By", + "Measures size of RPC request messages (uncompressed)", + ), + "rpc.server.response.size": ( + Histogram, + "By", + "Measures size of RPC response messages (uncompressed)", + ), + "rpc.server.requests_per_rpc": ( + Histogram, + "1", + "Measures the number of messages received per RPC. " + "Should be 1 for all non-streaming RPCs" + ), + "rpc.server.responses_per_rpc": ( + Histogram, + "1", + "Measures the number of messages sent per RPC. " + "Should be 1 for all non-streaming RPCs", + ), +} class TestOpenTelemetryServerInterceptor(TestBase): @@ -64,7 +76,18 @@ def assertEvent(self, event, name, attributes): self.assertEqual(event.name, name) for key, val in attributes.items(): self.assertIn(key, event.attributes, msg=str(event.attributes)) - self.assertEqual(val, event.attributes[key], msg=str(event.attributes)) + self.assertEqual( + val, event.attributes[key], msg=str(event.attributes) + ) + + def assertEqualMetricInstrumentationScope(self, scope_metrics, module): + self.assertEqual(scope_metrics.scope.name, module.__name__) + self.assertEqual(scope_metrics.scope.version, module.__version__) + + def assertMetricDataPointHasAttributes(self, data_point, attributes): + for key, val in attributes.items(): + self.assertIn(key, data_point.attributes) + self.assertEqual(val, data_point.attributes[key]) def test_instrumentor(self): @@ -75,17 +98,17 @@ def test_instrumentor(self): executor, options=(("grpc.so_reuseport", 0),), ) - add_GRPCTestServerServicer_to_server(Servicer(), server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - response = channel.unary_unary(rpc_call)(msg) + response = channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) @@ -110,9 +133,8 @@ def test_instrumentor(self): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -124,7 +146,7 @@ def test_instrumentor(self): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -149,22 +171,25 @@ def test_uninstrument(self): executor, options=(("grpc.so_reuseport", 0),), ) - add_GRPCTestServerServicer_to_server(Servicer(), server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - msg = Request().SerializeToString() + request_ser = Request().SerializeToString() try: server.start() - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) spans_list = self.memory_exporter.get_finished_spans() self.assertEqual(len(spans_list), 0) + # metrics_data = self.memory_metrics_reader.get_metrics_data() + # self.assertEqual(len(metrics_data.resource_metrics), 0) + def test_create_span(self): """Check that the interceptor wraps calls with spans server-side.""" @@ -177,17 +202,17 @@ def test_create_span(self): options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(Servicer(), server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - response = channel.unary_unary(rpc_call)(msg) + response = channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) @@ -213,9 +238,8 @@ def test_create_span(self): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -227,7 +251,7 @@ def test_create_span(self): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -275,11 +299,11 @@ def SimpleMethod(self, request, context): # setup the RPC rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - response = channel.unary_unary(rpc_call)(msg) + response = channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) @@ -306,9 +330,8 @@ def SimpleMethod(self, request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -320,7 +343,7 @@ def SimpleMethod(self, request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -347,6 +370,9 @@ def SimpleMethod(self, request, context): {} ) + logging.error("%r", self.memory_metrics_reader) + logging.error("%r", self.memory_metrics_reader.get_metrics_data()) + def test_create_span_streaming(self): """Check that the interceptor wraps calls with spans server-side, on a streaming call.""" @@ -360,18 +386,18 @@ def test_create_span_streaming(self): options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(Servicer(), server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - responses = list(channel.unary_stream(rpc_call)(msg)) + responses = list(channel.unary_stream(rpc_call)(request_ser)) finally: server.stop(None) @@ -397,9 +423,8 @@ def test_create_span_streaming(self): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -411,7 +436,7 @@ def test_create_span_streaming(self): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) for res_id, (event, response) in enumerate( @@ -462,11 +487,11 @@ def ServerStreamingMethod(self, request, context): # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - list(channel.unary_stream(rpc_call)(msg)) + list(channel.unary_stream(rpc_call)(request_ser)) finally: server.stop(None) @@ -493,9 +518,8 @@ def ServerStreamingMethod(self, request, context): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -518,7 +542,7 @@ def handler(request, context): active_span_in_handler = trace.get_current_span() return Response() - servicer = Servicer() + servicer = TestServer() servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=1) as executor: @@ -532,19 +556,19 @@ def handler(request, context): channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - msg = Request().SerializeToString() + request_ser = Request().SerializeToString() active_span_before_call = trace.get_current_span() try: server.start() - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) active_span_after_call = trace.get_current_span() self.assertEqual(active_span_before_call, trace.INVALID_SPAN) self.assertEqual(active_span_after_call, trace.INVALID_SPAN) - self.assertIsInstance(active_span_in_handler, trace_sdk.Span) + self.assertIsInstance(active_span_in_handler, trace.Span) self.assertIsNone(active_span_in_handler.parent) def test_span_lifetime_streaming(self): @@ -564,7 +588,7 @@ def handler(request, context): response_data=data, ) - servicer = Servicer() + servicer = TestServer() servicer.ServerStreamingMethod = handler with futures.ThreadPoolExecutor(max_workers=1) as executor: @@ -579,19 +603,19 @@ def handler(request, context): rpc_call = "/GRPCTestServer/ServerStreamingMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() active_span_before_call = trace.get_current_span() try: server.start() - list(channel.unary_stream(rpc_call)(msg)) + list(channel.unary_stream(rpc_call)(request_ser)) finally: server.stop(None) active_span_after_call = trace.get_current_span() self.assertEqual(active_span_before_call, trace.INVALID_SPAN) self.assertEqual(active_span_after_call, trace.INVALID_SPAN) - self.assertIsInstance(active_span_in_handler, trace_sdk.Span) + self.assertIsInstance(active_span_in_handler, trace.Span) self.assertIsNone(active_span_in_handler.parent) def test_sequential_server_spans(self): @@ -606,7 +630,7 @@ def handler(request, context): active_spans_in_handler.append(trace.get_current_span()) return Response() - servicer = Servicer() + servicer = TestServer() servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=1) as executor: @@ -620,12 +644,12 @@ def handler(request, context): channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - msg = Request().SerializeToString() + request_ser = Request().SerializeToString() try: server.start() - response_1 = channel.unary_unary(rpc_call)(msg) - response_2 = channel.unary_unary(rpc_call)(msg) + response_1 = channel.unary_unary(rpc_call)(request_ser) + response_2 = channel.unary_unary(rpc_call)(request_ser) finally: server.stop(None) @@ -650,9 +674,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -663,7 +686,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -696,7 +719,7 @@ def handler(request, context): active_spans_in_handler.append(trace.get_current_span()) return Response() - servicer = Servicer() + servicer = TestServer() servicer.SimpleMethod = handler with futures.ThreadPoolExecutor(max_workers=2) as executor: @@ -712,18 +735,18 @@ def handler(request, context): rpc_call = "/GRPCTestServer/SimpleMethod" request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request_ser = request.SerializeToString() try: server.start() - # Interleave calls so spans are active on each thread at the same - # time + # Interleave calls so spans are active on each thread at the + # same time with futures.ThreadPoolExecutor(max_workers=2) as tpe: f1 = tpe.submit( - channel.unary_unary(rpc_call), msg + channel.unary_unary(rpc_call), request_ser ) f2 = tpe.submit( - channel.unary_unary(rpc_call), msg + channel.unary_unary(rpc_call), request_ser ) futures.wait((f1, f2)) finally: @@ -750,9 +773,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0], }, ) @@ -764,7 +786,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -780,37 +802,29 @@ def handler(request, context): def test_abort(self): """Check that we can catch an abort properly""" + abort_message = "abort" + # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed failure message - failure_message = "This is a test failure" - - # aborting RPC handler - def handler(request, context): - context.abort(grpc.StatusCode.FAILED_PRECONDITION, failure_message) - - servicer = Servicer() - servicer.SimpleMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=abort_message) + request_ser = request.SerializeToString() server.start() # unfortunately, these are just bare exceptions in grpc... with self.assertRaises(Exception): - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) server.stop(None) spans_list = self.memory_exporter.get_finished_spans() @@ -829,7 +843,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + f"{grpc.StatusCode.FAILED_PRECONDITION}: {abort_message}", ) # check attributes @@ -842,9 +856,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0], }, ) @@ -856,49 +869,37 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) def test_abort_streaming(self): """Check that we can catch an abort of a streaming call properly""" + abort_message = "abort" + # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed failure message - failure_message = "This is a test failure" - - # aborting RPC handler - def handler(request, context): - yield Response( - server_id=request.client_id, - response_data="one", - ) - context.abort(grpc.StatusCode.FAILED_PRECONDITION, failure_message) - - servicer = Servicer() - servicer.ServerStreamingMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=abort_message) + request_ser = request.SerializeToString() server.start() responses = [] with self.assertRaises(Exception): - for res in channel.unary_stream(rpc_call)(msg): + for res in channel.unary_stream(rpc_call)(request_ser): responses.append(res) server.stop(None) @@ -918,7 +919,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + f"{grpc.StatusCode.FAILED_PRECONDITION}: {abort_message}", ) # check attributes @@ -931,9 +932,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0], }, ) @@ -946,7 +946,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -962,35 +962,29 @@ def handler(request, context): def test_cancel(self): """Check that we can catch a cancellation properly""" + cancel_message = "cancel" + # Intercept gRPC calls... interceptor = server_interceptor() - # aborting RPC handler - def handler(request, context): - context.cancel() - return Response() - - servicer = Servicer() - servicer.SimpleMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=cancel_message) + request_ser = request.SerializeToString() server.start() # unfortunately, these are just bare exceptions in grpc... with self.assertRaises(Exception): # as cm: - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) # exc = cm.exception server.stop(None) @@ -1027,9 +1021,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.CANCELLED.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], }, ) @@ -1041,7 +1034,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1055,42 +1048,33 @@ def handler(request, context): ) def test_cancel_streaming(self): - """Check that we can catch a cancellation of a streaming call properly""" + """Check that we can catch a cancellation of a streaming call properly. + """ + + cancel_message = "cancel" # Intercept gRPC calls... interceptor = server_interceptor() - # aborting RPC handler - def handler(request, context): - yield Response( - server_id=request.client_id, - response_data="one", - ) - context.cancel() - return - - servicer = Servicer() - servicer.ServerStreamingMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=cancel_message) + request_ser = request.SerializeToString() server.start() responses = [] with self.assertRaises(Exception): - for res in channel.unary_stream(rpc_call)(msg): + for res in channel.unary_stream(rpc_call)(request_ser): responses.append(res) server.stop(None) @@ -1127,9 +1111,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.CANCELLED.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], }, ) @@ -1142,7 +1125,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1158,39 +1141,29 @@ def handler(request, context): def test_error(self): """Check that we can catch an error properly""" + error_message = "error" + # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed failure message - failure_message = "This is a test failure" - - # error RPC handler - def handler(request, context): - context.set_code(grpc.StatusCode.FAILED_PRECONDITION) - context.set_details(failure_message) - return Response() - - servicer = Servicer() - servicer.SimpleMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=error_message) + request_ser = request.SerializeToString() server.start() # unfortunately, these are just bare exceptions in grpc... with self.assertRaises(Exception): - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) server.stop(None) spans_list = self.memory_exporter.get_finished_spans() @@ -1209,7 +1182,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + f"{grpc.StatusCode.INVALID_ARGUMENT}: {error_message}", ) # Check attributes @@ -1222,9 +1195,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0], }, ) @@ -1236,7 +1208,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1252,44 +1224,30 @@ def handler(request, context): def test_error_streaming(self): """Check that we can catch an error in a streaming call properly""" + error_message = "error" + # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed failure message - failure_message = "This is a test failure" - - # error RPC handler - def handler(request, context): - yield Response( - server_id=request.client_id, - response_data="one", - ) - context.set_code(grpc.StatusCode.FAILED_PRECONDITION) - context.set_details(failure_message) - return - - servicer = Servicer() - servicer.ServerStreamingMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=error_message) + request_ser = request.SerializeToString() server.start() responses = [] with self.assertRaises(Exception): - for res in channel.unary_stream(rpc_call)(msg): + for res in channel.unary_stream(rpc_call)(request_ser): responses.append(res) server.stop(None) @@ -1309,7 +1267,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{grpc.StatusCode.FAILED_PRECONDITION}: {failure_message}", + f"{grpc.StatusCode.INVALID_ARGUMENT}: {error_message}", ) # Check attributes @@ -1322,9 +1280,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.FAILED_PRECONDITION.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0], }, ) @@ -1337,7 +1294,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1353,37 +1310,29 @@ def handler(request, context): def test_raise_exception(self): """Check that we can catch a raised exception properly""" + exc_message = "exception" + # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed error message - err_message = "This is a value error" - - # error RPC handler - def handler(request, context): - raise ValueError(err_message) - - servicer = Servicer() - servicer.SimpleMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") rpc_call = "/GRPCTestServer/SimpleMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=exc_message) + request_ser = request.SerializeToString() server.start() # unfortunately, these are just bare exceptions in grpc... with self.assertRaises(Exception): - channel.unary_unary(rpc_call)(msg) + channel.unary_unary(rpc_call)(request_ser) server.stop(None) spans_list = self.memory_exporter.get_finished_spans() @@ -1402,7 +1351,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{ValueError.__name__}: {err_message}", + f"{ValueError.__name__}: {exc_message}", ) # Check attributes @@ -1415,9 +1364,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.UNKNOWN.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0], }, ) @@ -1429,7 +1377,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1437,51 +1385,41 @@ def handler(request, context): "exception", { "exception.type": ValueError.__name__, - "exception.message": err_message, + "exception.message": exc_message, # "exception.stacktrace": "...", "exception.escaped": str(False), } ) def test_raise_exception_streaming(self): - """Check that we can catch a raised exception in a streaming call properly""" + """Check that we can catch a raised exception in a streaming call + properly. + """ + + exc_message = "exception" # Intercept gRPC calls... interceptor = server_interceptor() - # our detailed error message - err_message = "This is a value error" - - # error RPC handler - def handler(request, context): - yield Response( - server_id=request.client_id, - response_data="one", - ) - raise ValueError(err_message) - - servicer = Servicer() - servicer.ServerStreamingMethod = handler - with futures.ThreadPoolExecutor(max_workers=1) as executor: server = grpc.server( executor, options=(("grpc.so_reuseport", 0),), interceptors=[interceptor], ) - add_GRPCTestServerServicer_to_server(servicer, server) + add_GRPCTestServerServicer_to_server(TestServer(), server) port = server.add_insecure_port("[::]:0") channel = grpc.insecure_channel(f"localhost:{port:d}") # setup the RPC rpc_call = "/GRPCTestServer/ServerStreamingMethod" - request = Request(client_id=1, request_data="test") - msg = request.SerializeToString() + request = Request(client_id=1, request_data=exc_message) + request_ser = request.SerializeToString() server.start() responses = [] with self.assertRaises(Exception): - for res in channel.unary_stream(rpc_call)(msg): + for res in channel.unary_stream(rpc_call)(request_ser): responses.append(res) server.stop(None) @@ -1501,7 +1439,7 @@ def handler(request, context): self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( span.status.description, - f"{ValueError.__name__}: {err_message}", + f"{ValueError.__name__}: {exc_message}", ) # Check attributes @@ -1514,9 +1452,8 @@ def handler(request, context): SpanAttributes.RPC_METHOD: "ServerStreamingMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.UNKNOWN.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0], }, ) @@ -1529,7 +1466,7 @@ def handler(request, context): { SpanAttributes.MESSAGE_TYPE: "RECEIVED", SpanAttributes.MESSAGE_ID: 1, - SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(msg) + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: len(request_ser) } ) self.assertEvent( @@ -1546,12 +1483,736 @@ def handler(request, context): "exception", { "exception.type": ValueError.__name__, - "exception.message": err_message, + "exception.message": exc_message, # "exception.stacktrace": "...", "exception.escaped": str(False), } ) + def test_metrics(self): + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + request_ser = request.SerializeToString() + + try: + server.start() + response = channel.unary_unary(rpc_call)(request_ser) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, 1) + if metric.name == "rpc.server.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.sum, len(request_ser)) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.sum, len(response)) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_error(self): + + error_message = "error" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data=error_message) + request_ser = request.SerializeToString() + + try: + server.start() + with self.assertRaises(grpc.RpcError): + channel.unary_unary(rpc_call)(request_ser) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, 1) + if metric.name == "rpc.server.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.sum, len(request_ser)) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.sum, 0) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[ + 0 + ] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_three_calls(self): + no_calls = 3 + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + rpc_call = "/GRPCTestServer/SimpleMethod" + request = Request(client_id=1, request_data="test") + request_ser = request.SerializeToString() + + try: + server.start() + for _ in range(no_calls): + response = channel.unary_unary(rpc_call)(request_ser) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, no_calls) + if metric.name == "rpc.server.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual( + point.sum, no_calls * len(request_ser) + ) + elif metric.name == "rpc.server.response.size": + self.assertEqual( + point.sum, no_calls * len(response) + ) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.sum, no_calls) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.sum, no_calls) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_client_streaming(self): + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ClientStreamingMethod" + request = Request(client_id=1, request_data="test") + requests = [request.SerializeToString() for _ in range(5)] + + try: + server.start() + response = channel.stream_unary(rpc_call)(iter(requests)) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.server.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, sum(map(len, requests)) + ) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(response)) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: + "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_client_streaming_abort(self): + + error_message = "abort" + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ClientStreamingMethod" + request = Request(client_id=1, request_data=error_message) + requests = [request.SerializeToString() for _ in range(5)] + + try: + server.start() + with self.assertRaises(grpc.RpcError): + channel.stream_unary(rpc_call)(iter(requests)) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertLess( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.server.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, sum(map(len, requests)) + ) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.count, 0) + # self.assertEqual(point.sum, 0) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.count, 0) + # self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: + "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[ + 0 + ] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_server_streaming(self): + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/ServerStreamingMethod" + request = Request(client_id=1, request_data="test") + request_ser = request.SerializeToString() + + try: + server.start() + responses = list(channel.unary_stream(rpc_call)(request_ser)) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.server.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(request_ser)) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, sum(map(len, responses)) + ) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: + "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + + def test_metrics_bidirectional_streaming(self): + + # Intercept gRPC calls... + interceptor = server_interceptor() + + with futures.ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server( + executor, + options=(("grpc.so_reuseport", 0),), + interceptors=[interceptor], + ) + add_GRPCTestServerServicer_to_server(TestServer(), server) + port = server.add_insecure_port("[::]:0") + channel = grpc.insecure_channel(f"localhost:{port:d}") + + # setup the RPC + rpc_call = "/GRPCTestServer/BidirectionalStreamingMethod" + request = Request(client_id=1, request_data="test") + requests = [request.SerializeToString() for _ in range(5)] + + try: + server.start() + responses = list( + channel.stream_stream(rpc_call)(iter(requests)) + ) + finally: + server.stop(None) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + data_point_seen = True + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.server.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.server.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, sum(map(len, requests)) + ) + elif metric.name == "rpc.server.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, sum(map(len, responses)) + ) + elif metric.name == "rpc.server.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.server.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.NET_PEER_IP: "[::1]", + # SpanAttributes.NET_PEER_PORT: "0", + SpanAttributes.NET_PEER_NAME: "localhost", + SpanAttributes.RPC_METHOD: + "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.server.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + data_point_seen &= True + + self.assertTrue(data_point_seen) + def get_latch(num): """Get a countdown latch function for use in n threads.""" From eda84c97f94fd9199a6d2c81945bb40443d60864 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:44:01 +0200 Subject: [PATCH 11/17] moved openetlemetry service context into utilities --- .../instrumentation/grpc/_server.py | 178 ++---------- .../instrumentation/grpc/_utilities.py | 255 ++++++++++++++---- 2 files changed, 224 insertions(+), 209 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py index c070b99232..47297c7aa8 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_server.py @@ -24,14 +24,14 @@ import copy import logging from contextlib import contextmanager -from typing import Callable, Dict, Iterable, Iterator, Generator, NoReturn, Optional +from typing import Callable, Iterator, Generator, Optional import grpc from opentelemetry import metrics, trace from opentelemetry.context import attach, detach -from opentelemetry.instrumentation.grpc._types import Metadata, ProtoMessage, ProtoMessageOrIterator -from opentelemetry.instrumentation.grpc._utilities import _EventMetricRecorder, _MetricKind +from opentelemetry.instrumentation.grpc._types import ProtoMessage, ProtoMessageOrIterator +from opentelemetry.instrumentation.grpc._utilities import _EventMetricRecorder, _MetricKind, _OpenTelemetryServicerContext from opentelemetry.propagate import extract from opentelemetry.semconv.trace import MessageTypeValues, RpcSystemValues, SpanAttributes from opentelemetry.trace.status import Status, StatusCode @@ -79,150 +79,6 @@ def _wrap_rpc_behavior( ) -# pylint:disable=abstract-method -class _OpenTelemetryServicerContext(grpc.ServicerContext): - - def __init__( - self, - servicer_context: grpc.ServicerContext, - active_span: trace.Span - ) -> None: - self._servicer_context = servicer_context - self._active_span = active_span - self._code = grpc.StatusCode.OK - self._details = None - super().__init__() - - def __getattr__(self, attr): - return getattr(self._servicer_context, attr) - - # Interface of grpc.RpcContext - - # pylint: disable=invalid-name - def add_callback(self, fn: Callable[[], None]) -> None: - return self._servicer_context.add_callback(fn) - - def cancel(self) -> None: - self._code = grpc.StatusCode.CANCELLED - self._details = grpc.StatusCode.CANCELLED.value[1] - self._active_span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, self._code.value[0] - ) - self._active_span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{self._code}: {self._details}", - ) - ) - return self._servicer_context.cancel() - - def is_active(self) -> bool: - return self._servicer_context.is_active() - - def time_remaining(self) -> Optional[float]: - return self._servicer_context.time_remaining() - - # Interface of grpc.ServicerContext - - def abort(self, code: grpc.StatusCode, details: str) -> NoReturn: - if not hasattr(self._servicer_context, "abort"): - raise RuntimeError( - "abort() is not supported with the installed version of grpcio" - ) - self._code = code - self._details = details - self._active_span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] - ) - self._active_span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{code}: {details}", - ) - ) - return self._servicer_context.abort(code, details) - - def abort_with_status(self, status: grpc.Status) -> NoReturn: - if not hasattr(self._servicer_context, "abort_with_status"): - raise RuntimeError( - "abort_with_status() is not supported with the installed " - "version of grpcio" - ) - return self._servicer_context.abort_with_status(status) - - def auth_context(self) -> Dict[str, Iterable[bytes]]: - return self._servicer_context.auth_context() - - def code(self) -> grpc.StatusCode: - if not hasattr(self._servicer_context, "code"): - raise RuntimeError( - "code() is not supported with the installed version of grpcio" - ) - return self._servicer_context.code() - - def details(self) -> str: - if not hasattr(self._servicer_context, "details"): - raise RuntimeError( - "details() is not supported with the installed version of " - "grpcio" - ) - return self._servicer_context.details() - - def disable_next_message_compression(self) -> None: - return self._service_context.disable_next_message_compression() - - def invocation_metadata(self) -> Metadata: - return self._servicer_context.invocation_metadata() - - def peer(self) -> str: - return self._servicer_context.peer() - - def peer_identities(self) -> Optional[Iterable[bytes]]: - return self._servicer_context.peer_identities() - - def peer_identity_key(self) -> Optional[str]: - return self._servicer_context.peer_identity_key() - - def send_initial_metadata(self, initial_metadata: Metadata) -> None: - return self._servicer_context.send_initial_metadata(initial_metadata) - - def set_code(self, code: grpc.StatusCode) -> None: - self._code = code - # use details if we already have it, otherwise the status description - details = self._details or code.value[1] - self._active_span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] - ) - if code != grpc.StatusCode.OK: - self._active_span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{code}: {details}", - ) - ) - return self._servicer_context.set_code(code) - - def set_compression(self, compression: grpc.Compression) -> None: - return self._servicer_context.set_compression(compression) - - def set_details(self, details: str) -> None: - self._details = details - if self._code != grpc.StatusCode.OK: - self._active_span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{self._code}: {details}", - ) - ) - return self._servicer_context.set_details(details) - - def set_trailing_metadata(self, trailing_metadata: Metadata) -> None: - return self._servicer_context.set_trailing_metadata(trailing_metadata) - - def trailing_metadata(self) -> Metadata: - return self._servicer_context.trailing_metadata() - - # pylint:disable=abstract-method # pylint:disable=no-self-use # pylint:disable=unused-argument @@ -378,6 +234,7 @@ def intercept_unary_unary( context: grpc.ServicerContext, full_method: str ) -> ProtoMessage: + with self._set_remote_context(context): metric_attributes = self._create_attributes(context, full_method) span_attributes = copy.deepcopy(metric_attributes) @@ -393,12 +250,11 @@ def intercept_unary_unary( record_exception=False, set_status_on_exception=False ) as span: - with self._record_duration_manager(metric_attributes, context): + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + with self._record_duration_manager(metric_attributes, context): try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - # record the request self._record_unary_request( span, @@ -448,6 +304,7 @@ def intercept_unary_stream( context: grpc.ServicerContext, full_method: str ) -> Iterator[ProtoMessage]: + with self._set_remote_context(context): metric_attributes = self._create_attributes(context, full_method) span_attributes = copy.deepcopy(metric_attributes) @@ -463,12 +320,11 @@ def intercept_unary_stream( record_exception=False, set_status_on_exception=False ) as span: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) with self._record_duration_manager(metric_attributes, context): try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - # record the request self._record_unary_request( span, @@ -516,6 +372,7 @@ def intercept_stream_unary( context: grpc.ServicerContext, full_method: str ) -> ProtoMessage: + with self._set_remote_context(context): metric_attributes = self._create_attributes(context, full_method) span_attributes = copy.deepcopy(metric_attributes) @@ -531,12 +388,11 @@ def intercept_stream_unary( record_exception=False, set_status_on_exception=False ) as span: - with self._record_duration_manager(metric_attributes, context): + # wrap the context + context = _OpenTelemetryServicerContext(context, span) + with self._record_duration_manager(metric_attributes, context): try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - # wrap the request iterator with a recorder request_iterator = self._record_streaming_request( span, @@ -587,6 +443,7 @@ def intercept_stream_stream( context: grpc.ServicerContext, full_method: str ) -> Iterator[ProtoMessage]: + with self._set_remote_context(context): metric_attributes = self._create_attributes(context, full_method) span_attributes = copy.deepcopy(metric_attributes) @@ -602,12 +459,11 @@ def intercept_stream_stream( record_exception=False, set_status_on_exception=False ) as span: + # wrap the context + context = _OpenTelemetryServicerContext(context, span) with self._record_duration_manager(metric_attributes, context): try: - # wrap the context - context = _OpenTelemetryServicerContext(context, span) - # wrap the request iterator with a recorder request_iterator = self._record_streaming_request( span, diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py index 543254c396..3c63f3f2b8 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py @@ -18,14 +18,14 @@ from contextlib import contextmanager from enum import Enum from timeit import default_timer -from typing import Generator, Iterator +from typing import Callable, Dict, Generator, Iterable, Iterator, NoReturn, Optional import grpc -from opentelemetry.instrumentation.grpc._types import ProtoMessage -from opentelemetry.metrics import Meter -from opentelemetry.trace import Span +from opentelemetry import metrics, trace +from opentelemetry.instrumentation.grpc._types import ProtoMessage from opentelemetry.semconv.trace import MessageTypeValues, SpanAttributes -from opentelemetry.util.types import Attributes +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util.types import Attributes, Metadata _MESSAGE: str = "message" @@ -33,7 +33,7 @@ def _add_message_event( - active_span: Span, + active_span: trace.Span, message_type: str, message_size_by: int, message_id: int = 1 @@ -41,11 +41,11 @@ def _add_message_event( """Adds a message event of an RPC to an active span. Args: - active_span (Span): The active span in which to record the message - as event. + active_span (trace.Span): The active span in which to record the + message as event. message_type (str): The message type value as str, either "SENT" or "RECEIVED". - message_size_by (int): The (uncompressed) message size in bytes as int. + message_size_by (int): The (uncompressed) message size in bytes. message_id (int, optional): The message ID. Defaults to 1. """ @@ -59,6 +59,33 @@ def _add_message_event( ) +def _get_status_code(context: grpc.RpcContext) -> grpc.StatusCode: + """Extracts the status code from a context, even though the context is a + ServicerContext of a grpc version which does not support code(). + + Args: + context (grpc.RpcContext): The context to extract the status code from. + + Returns: + grpc.StatusCode: The extracted status code. + """ + + try: + code = context.code() + except RuntimeError: + if isinstance(code, _OpenTelemetryServicerContext): + code = context._code + elif isinstance(code, grpc.ServicerContext): + code = context._state._code + else: + raise + + if code is not None: + return code + + return grpc.StatusCode.OK + + class _ClientCallDetails( namedtuple( "_ClientCallDetails", @@ -86,7 +113,7 @@ class _EventMetricRecorder: and for recording the duration of a RPC. """ - def __init__(self, meter: Meter, kind: _MetricKind) -> None: + def __init__(self, meter: metrics.Meter, kind: _MetricKind) -> None: """Initializes the _EventMetricRecorder. Args: @@ -129,7 +156,7 @@ def __init__(self, meter: Meter, kind: _MetricKind) -> None: def _record_unary_request( self, - active_span: Span, + active_span: trace.Span, request: ProtoMessage, message_type: MessageTypeValues, metric_attributes: Attributes @@ -141,8 +168,8 @@ def _record_unary_request( histogram. Args: - active_span (Span): The active span in which to record the request - as event. + active_span (trace.Span): The active span in which to record the + request message as event. request (ProtoMessage): The request message. message_type (MessageTypeValues): The message type value. metric_attributes (Attributes): The attributes to record in the @@ -154,22 +181,22 @@ def _record_unary_request( self._request_size_histogram.record(message_size_by, metric_attributes) self._requests_per_rpc_histogram.record(1, metric_attributes) - def _record_response( + def _record_unary_or_streaming_response( self, - active_span: Span, + active_span: trace.Span, response: ProtoMessage, message_type: MessageTypeValues, metric_attributes: Attributes, response_id: int = 1 ) -> None: - """Records a unary OR streaming response. + """Records a unary OR a single, streaming response. The response is recorded as event and its size in the response-size- histogram. Args: - active_span (Span): The active span in which to record the response - as event. + active_span (trace.Span): The active span in which to record the + response message as event. response (ProtoMessage): The response message. message_type (MessageTypeValues): The message type value. metric_attributes (Attributes): The attributes to record in the @@ -188,7 +215,7 @@ def _record_response( message_size_by, metric_attributes ) - def _record_responses_per_rpc( + def _record_num_of_responses_per_rpc( self, responses_per_rpc: int, metric_attributes: Attributes @@ -196,7 +223,6 @@ def _record_responses_per_rpc( """Records the number of responses in the responses-per-RPC-histogram for a streaming response. - Args: responses_per_rpc (int): The number of responses. metric_attributes (Attributes): The attributes to record in the @@ -209,7 +235,7 @@ def _record_responses_per_rpc( def _record_unary_response( self, - active_span: Span, + active_span: trace.Span, response: ProtoMessage, message_type: MessageTypeValues, metric_attributes: Attributes @@ -221,22 +247,22 @@ def _record_unary_response( histogram. Args: - active_span (Span): The active span in which to record the response - as event. + active_span (trace.Span): The active span in which to record the + response message as event. response (ProtoMessage): The response message. message_type (MessageTypeValues): The message type value. metric_attributes (Attributes): The attributes to record in the metrics. """ - self._record_response( + self._record_unary_or_streaming_response( active_span, response, message_type, metric_attributes ) - self._record_responses_per_rpc(1, metric_attributes) + self._record_num_of_responses_per_rpc(1, metric_attributes) def _record_streaming_request( self, - active_span: Span, + active_span: trace.Span, request_iterator: Iterator[ProtoMessage], message_type: MessageTypeValues, metric_attributes: Attributes @@ -248,8 +274,8 @@ def _record_streaming_request( histogram. Args: - active_span (Span): The active span in which to record the request - as event. + active_span (trace.Span): The active span in which to record the + request messages as event. request_iterator (Iterator[ProtoMessage]): The iterator over the request messages. message_type (MessageTypeValues): The message type value. @@ -280,7 +306,7 @@ def _record_streaming_request( def _record_streaming_response( self, - active_span: Span, + active_span: trace.Span, response_iterator: Iterator[ProtoMessage], message_type: MessageTypeValues, metric_attributes: Attributes @@ -292,8 +318,8 @@ def _record_streaming_response( histogram. Args: - active_span (Span): The active span in which to record the response - as event. + active_span (trace.Span): The active span in which to record the + response messages as event. response_iterator (Iterator[ProtoMessage]): The iterator over the response messages. message_type (MessageTypeValues): The message type value. @@ -308,7 +334,7 @@ def _record_streaming_response( try: res_id = 0 for res_id, response in enumerate(response_iterator, start=1): - self._record_response( + self._record_unary_or_streaming_response( active_span, response, message_type, @@ -317,7 +343,7 @@ def _record_streaming_response( ) yield response finally: - self._record_responses_per_rpc(res_id, metric_attributes) + self._record_num_of_responses_per_rpc(res_id, metric_attributes) def _start_duration_measurement(self) -> float: """Starts a duration measurement and returns the start time. @@ -347,14 +373,8 @@ def _record_duration( """ duration = max(round((default_timer() - start_time) * 1000), 0) - if context.code() in (None, grpc.StatusCode.OK): - metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( - grpc.StatusCode.OK.value[0] - ) - else: - metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( - context.code().value[0] - ) + code = _get_status_code(context) + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = code.value[0] self._duration_histogram.record(duration, metric_attributes) @contextmanager @@ -381,12 +401,151 @@ def _record_duration_manager( yield finally: duration = max(round((default_timer() - start_time) * 1000), 0) - if context.code() in (None, grpc.StatusCode.OK): - metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( - grpc.StatusCode.OK.value[0] + code = _get_status_code(context) + metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( + code.value[0] + ) + self._duration_histogram.record(duration, metric_attributes) + + +# pylint:disable=abstract-method +class _OpenTelemetryServicerContext(grpc.ServicerContext): + + def __init__( + self, + servicer_context: grpc.ServicerContext, + active_span: trace.Span + ) -> None: + self._servicer_context = servicer_context + self._active_span = active_span + self._code = grpc.StatusCode.OK + self._details = None + super().__init__() + + def __getattr__(self, attr): + return getattr(self._servicer_context, attr) + + # Interface of grpc.RpcContext + + def add_callback(self, callback: Callable[[], None]) -> None: + return self._servicer_context.add_callback(callback) + + def cancel(self) -> bool: + self._code = grpc.StatusCode.CANCELLED + self._details = grpc.StatusCode.CANCELLED.value[1] + self._active_span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, self._code.value[0] + ) + self._active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{self._code}: {self._details}", + ) + ) + return self._servicer_context.cancel() + + def is_active(self) -> bool: + return self._servicer_context.is_active() + + def time_remaining(self) -> Optional[float]: + return self._servicer_context.time_remaining() + + # Interface of grpc.ServicerContext + + def abort(self, code: grpc.StatusCode, details: str) -> NoReturn: + if not hasattr(self._servicer_context, "abort"): + raise RuntimeError( + "abort() is not supported with the installed version of grpcio" + ) + self._code = code + self._details = details + self._active_span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] + ) + self._active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{code}: {details}", + ) + ) + return self._servicer_context.abort(code, details) + + def abort_with_status(self, status: grpc.Status) -> NoReturn: + if not hasattr(self._servicer_context, "abort_with_status"): + raise RuntimeError( + "abort_with_status() is not supported with the installed " + "version of grpcio" + ) + return self._servicer_context.abort_with_status(status) + + def auth_context(self) -> Dict[str, Iterable[bytes]]: + return self._servicer_context.auth_context() + + def code(self) -> grpc.StatusCode: + if not hasattr(self._servicer_context, "code"): + raise RuntimeError( + "code() is not supported with the installed version of grpcio" + ) + return self._servicer_context.code() + + def details(self) -> str: + if not hasattr(self._servicer_context, "details"): + raise RuntimeError( + "details() is not supported with the installed version of " + "grpcio" + ) + return self._servicer_context.details() + + def disable_next_message_compression(self) -> None: + return self._service_context.disable_next_message_compression() + + def invocation_metadata(self) -> Metadata: + return self._servicer_context.invocation_metadata() + + def peer(self) -> str: + return self._servicer_context.peer() + + def peer_identities(self) -> Optional[Iterable[bytes]]: + return self._servicer_context.peer_identities() + + def peer_identity_key(self) -> Optional[str]: + return self._servicer_context.peer_identity_key() + + def send_initial_metadata(self, initial_metadata: Metadata) -> None: + return self._servicer_context.send_initial_metadata(initial_metadata) + + def set_code(self, code: grpc.StatusCode) -> None: + self._code = code + # use details if we already have it, otherwise the status description + details = self._details or code.value[1] + self._active_span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] + ) + if code != grpc.StatusCode.OK: + self._active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{code}: {details}", ) - else: - metric_attributes[SpanAttributes.RPC_GRPC_STATUS_CODE] = ( - context.code().value[0] + ) + return self._servicer_context.set_code(code) + + def set_compression(self, compression: grpc.Compression) -> None: + return self._servicer_context.set_compression(compression) + + def set_details(self, details: str) -> None: + self._details = details + if self._code != grpc.StatusCode.OK: + self._active_span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{self._code}: {details}", ) - self._duration_histogram.record(duration, metric_attributes) + ) + return self._servicer_context.set_details(details) + + def set_trailing_metadata(self, trailing_metadata: Metadata) -> None: + return self._servicer_context.set_trailing_metadata(trailing_metadata) + + def trailing_metadata(self) -> Metadata: + return self._servicer_context.trailing_metadata() From d252e7155386c3558044ebb5aa06a6a4b330ce6e Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:44:19 +0200 Subject: [PATCH 12/17] added streaming redezvous type --- .../src/opentelemetry/instrumentation/grpc/_types.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py index 81f12ba943..c618c580a1 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_types.py @@ -14,6 +14,7 @@ """Internal types.""" +import abc from typing import Dict, Iterator, TypeVar, Union import grpc @@ -27,3 +28,13 @@ ProtoMessageOrIterator = Union[ProtoMessage, Iterator[ProtoMessage]] """Type for a protobuf message or a iterator of protobuf messages.""" + + +class StreamingRendezvous( + grpc.Call, + grpc.Future, + grpc.RpcError, + Iterator, + abc.ABC +): + """Class for a streaming rendezvous.""" From 9f1792f98d86a13e41b83f2f80a2b591e3f2cf3c Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:55:01 +0200 Subject: [PATCH 13/17] Complete redesign of OpenTelemetryClientInterceptor including support of metrics, tracing of messages and changed API to officical grpc.ClientInterceptor API --- .../instrumentation/grpc/__init__.py | 46 ++- .../instrumentation/grpc/_client.py | 368 +++++++++++++----- .../instrumentation/grpc/_utilities.py | 146 ++++++- .../instrumentation/grpc/grpcext/__init__.py | 125 ------ .../grpc/grpcext/_interceptor.py | 350 ----------------- 5 files changed, 445 insertions(+), 590 deletions(-) delete mode 100644 instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/__init__.py delete mode 100644 instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/_interceptor.py diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py index f67f938d53..41d97fb461 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/__init__.py @@ -27,6 +27,11 @@ from opentelemetry import trace from opentelemetry.instrumentation.grpc import GrpcInstrumentorClient + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( ConsoleSpanExporter, @@ -38,6 +43,12 @@ except ImportError: from gen import helloworld_pb2, helloworld_pb2_grpc + exporter = ConsoleMetricExporter() + reader = PeriodicExportingMetricReader(exporter) + metrics.set_meter_provider( + MeterProvider(metric_readers=[reader]) + ) + trace.set_tracer_provider(TracerProvider()) trace.get_tracer_provider().add_span_processor( SimpleSpanProcessor(ConsoleSpanExporter()) @@ -58,6 +69,18 @@ def run(): logging.basicConfig() run() +You can also add the instrumentor manually, rather than using +:py:class:`~opentelemetry.instrumentation.grpc.GrpcInstrumentorClient`: + +.. code-block:: python + + from opentelemetry.instrumentation.grpc import client_interceptor + + channel = grpc.intercept_channel( + grpc.insecure_channel(...) / grpc.secure_channel(...), + client_interceptor() + ) + Usage Server ------------ .. code-block:: python @@ -135,7 +158,6 @@ def serve(): from wrapt import wrap_function_wrapper as _wrap from opentelemetry import metrics, trace -from opentelemetry.instrumentation.grpc.grpcext import intercept_channel from opentelemetry.instrumentation.grpc.package import _instruments from opentelemetry.instrumentation.grpc.version import __version__ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor @@ -235,39 +257,47 @@ def _uninstrument(self, **kwargs): def wrapper_fn(self, original_func, instance, args, kwargs): channel = original_func(*args, **kwargs) + meter_provider = kwargs.get("meter_provider") tracer_provider = kwargs.get("tracer_provider") - return intercept_channel( + return grpc.intercept_channel( channel, - client_interceptor(tracer_provider=tracer_provider), + client_interceptor( + meter_provider=meter_provider, + tracer_provider=tracer_provider + ) ) -def client_interceptor(tracer_provider=None): +def client_interceptor(meter_provider=None, tracer_provider=None): """Create a gRPC client channel interceptor. Args: - tracer: The tracer to use to create client-side spans. + meter_provider: The meter provider which allows access to the meter. + tracer_provider: The tracer provider which allows access to the tracer. Returns: An invocation-side interceptor object. """ + from . import _client + meter = metrics.get_meter(__name__, __version__, meter_provider) tracer = trace.get_tracer(__name__, __version__, tracer_provider) - return _client.OpenTelemetryClientInterceptor(tracer) + return _client.OpenTelemetryClientInterceptor(meter, tracer) def server_interceptor(meter_provider=None, tracer_provider=None): """Create a gRPC server interceptor. Args: - meter_provider: The meter provider which allows acess to the meter. - tracer_provider: The tracer provider which allows acess to the tracer. + meter_provider: The meter provider which allows access to the meter. + tracer_provider: The tracer provider which allows access to the tracer. Returns: A service-side interceptor object. """ + from . import _server meter = metrics.get_meter(__name__, __version__, meter_provider) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_client.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_client.py index b73b822b14..e36441bdb2 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_client.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_client.py @@ -20,18 +20,20 @@ """Implementation of the invocation-side open-telemetry interceptor.""" from collections import OrderedDict -from typing import MutableMapping +from typing import Callable, Iterator, MutableMapping import grpc from opentelemetry import context, trace -from opentelemetry.instrumentation.grpc import grpcext -from opentelemetry.instrumentation.grpc._utilities import RpcInfo +from opentelemetry.instrumentation.grpc._types import ProtoMessage +from opentelemetry.instrumentation.grpc._utilities import _ClientCallDetails, _EventMetricRecorder, _MetricKind, _OpentelemetryResponseIterator from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY from opentelemetry.propagate import inject from opentelemetry.propagators.textmap import Setter -from opentelemetry.semconv.trace import SpanAttributes +from opentelemetry.semconv.trace import MessageTypeValues, RpcSystemValues, SpanAttributes +from opentelemetry.trace import Span from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util.types import Attributes class _CarrierSetter(Setter): @@ -39,160 +41,316 @@ class _CarrierSetter(Setter): keys as is required by grpc. """ - def set(self, carrier: MutableMapping[str, str], key: str, value: str): + def set( + self, + carrier: MutableMapping[str, str], + key: str, + value: str + ) -> None: carrier[key.lower()] = value _carrier_setter = _CarrierSetter() -def _make_future_done_callback(span, rpc_info): - def callback(response_future): - with trace.use_span(span, end_on_exit=True): +def _make_future_done_callback( + span: Span, + metric_recorder: _EventMetricRecorder, + attributes: Attributes, + start_time: float, +) -> Callable[[grpc.Future], None]: + + def callback(response_future: grpc.Future) -> None: + with trace.use_span( + span, + record_exception=False, + set_status_on_exception=False, + end_on_exit=True, + ): code = response_future.code() if code != grpc.StatusCode.OK: - rpc_info.error = code - return - response = response_future.result() - rpc_info.response = response + details = response_future.details() + span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] + ) + span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{code}: {details}", + ) + ) + + try: + span.record_exception(response_future.exception()) + except grpc.FutureCancelledError: + pass + + else: + response = response_future.result() + if response is not None: + metric_recorder._record_unary_response( + span, response, MessageTypeValues.RECEIVED, attributes + ) + + metric_recorder._record_duration( + start_time, + attributes, + response_future + ) return callback class OpenTelemetryClientInterceptor( - grpcext.UnaryClientInterceptor, grpcext.StreamClientInterceptor + _EventMetricRecorder, + grpc.UnaryUnaryClientInterceptor, + grpc.UnaryStreamClientInterceptor, + grpc.StreamUnaryClientInterceptor, + grpc.StreamStreamClientInterceptor ): - def __init__(self, tracer): + + def __init__(self, meter, tracer): + super().__init__(meter, _MetricKind.CLIENT) self._tracer = tracer - def _start_span(self, method, **kwargs): - service, meth = method.lstrip("/").split("/", 1) - attributes = { - SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], - SpanAttributes.RPC_METHOD: meth, + def _create_attributes(self, full_method: str) -> Attributes: + service, method = full_method.lstrip("/").split("/", 1) + return { + SpanAttributes.RPC_SYSTEM: RpcSystemValues.GRPC.value, SpanAttributes.RPC_SERVICE: service, + SpanAttributes.RPC_METHOD: method, + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], } - return self._tracer.start_as_current_span( - name=method, - kind=trace.SpanKind.CLIENT, - attributes=attributes, - **kwargs, - ) - - # pylint:disable=no-self-use - def _trace_result(self, span, rpc_info, result): - # If the RPC is called asynchronously, add a callback to end the span - # when the future is done, else end the span immediately - if isinstance(result, grpc.Future): - result.add_done_callback( - _make_future_done_callback(span, rpc_info) - ) - return result - response = result - # Handle the case when the RPC is initiated via the with_call - # method and the result is a tuple with the first element as the - # response. - # http://www.grpc.io/grpc/python/grpc.html#grpc.UnaryUnaryMultiCallable.with_call - if isinstance(result, tuple): - response = result[0] - rpc_info.response = response - span.end() - return result - - def _intercept(self, request, metadata, client_info, invoker): + def intercept_unary_unary( + self, + continuation, + client_call_details: grpc.ClientCallDetails, + request: ProtoMessage + ): if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return invoker(request, metadata) + return continuation(client_call_details, request) + + attributes = self._create_attributes(client_call_details.method) - if not metadata: + if not client_call_details.metadata: mutable_metadata = OrderedDict() else: - mutable_metadata = OrderedDict(metadata) - with self._start_span( - client_info.full_method, + mutable_metadata = OrderedDict(client_call_details.metadata) + + with self._tracer.start_as_current_span( + name=client_call_details.method, + kind=trace.SpanKind.CLIENT, + attributes=attributes, end_on_exit=False, record_exception=False, set_status_on_exception=False, ) as span: - result = None + response_future = None + start_time = 0.0 + try: inject(mutable_metadata, setter=_carrier_setter) metadata = tuple(mutable_metadata.items()) - - rpc_info = RpcInfo( - full_method=client_info.full_method, + new_client_call_details = _ClientCallDetails( + method=client_call_details.method, + timeout=client_call_details.timeout, metadata=metadata, - timeout=client_info.timeout, - request=request, + credentials=client_call_details.credentials, + wait_for_ready=client_call_details.wait_for_ready, + compression=client_call_details.compression ) - result = invoker(request, metadata) - except Exception as exc: - if isinstance(exc, grpc.RpcError): - span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, - exc.code().value[0], - ) - span.set_status( - Status( - status_code=StatusCode.ERROR, - description=f"{type(exc).__name__}: {exc}", - ) + start_time = self._start_duration_measurement() + self._record_unary_request( + span, + request, + MessageTypeValues.SENT, + attributes ) - span.record_exception(exc) - raise exc + + response_future = continuation( + new_client_call_details, + request + ) + finally: - if not result: + if not response_future: span.end() - return self._trace_result(span, rpc_info, result) + else: + response_future.add_done_callback( + _make_future_done_callback( + span, + self, + attributes, + start_time + ) + ) - def intercept_unary(self, request, metadata, client_info, invoker): - return self._intercept(request, metadata, client_info, invoker) + return response_future - # For RPCs that stream responses, the result can be a generator. To record - # the span across the generated responses and detect any errors, we wrap - # the result in a new generator that yields the response values. - def _intercept_server_stream( - self, request_or_iterator, metadata, client_info, invoker + def intercept_unary_stream( + self, + continuation, + client_call_details: grpc.ClientCallDetails, + request: ProtoMessage ): - if not metadata: + if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return continuation(client_call_details, request) + + attributes = self._create_attributes(client_call_details.method) + + if not client_call_details.metadata: mutable_metadata = OrderedDict() else: - mutable_metadata = OrderedDict(metadata) + mutable_metadata = OrderedDict(client_call_details.metadata) - with self._start_span(client_info.full_method) as span: + with self._tracer.start_as_current_span( + name=client_call_details.method, + kind=trace.SpanKind.CLIENT, + attributes=attributes, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ) as span: inject(mutable_metadata, setter=_carrier_setter) metadata = tuple(mutable_metadata.items()) - rpc_info = RpcInfo( - full_method=client_info.full_method, + new_client_call_details = _ClientCallDetails( + method=client_call_details.method, + timeout=client_call_details.timeout, metadata=metadata, - timeout=client_info.timeout, + credentials=client_call_details.credentials, + wait_for_ready=client_call_details.wait_for_ready, + compression=client_call_details.compression ) - if client_info.is_client_stream: - rpc_info.request = request_or_iterator + start_time = self._start_duration_measurement() + self._record_unary_request( + span, request, MessageTypeValues.SENT, attributes + ) + + response_iterator = continuation( + new_client_call_details, request + ) + + return _OpentelemetryResponseIterator( + response_iterator, + self, span, + attributes, + start_time + ) + + def intercept_stream_unary( + self, + continuation, + client_call_details: grpc.ClientCallDetails, + request_iterator: Iterator[ProtoMessage] + ): + if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return continuation(client_call_details, request_iterator) + + attributes = self._create_attributes(client_call_details.method) + + if not client_call_details.metadata: + mutable_metadata = OrderedDict() + else: + mutable_metadata = OrderedDict(client_call_details.metadata) + + with self._tracer.start_as_current_span( + name=client_call_details.method, + kind=trace.SpanKind.CLIENT, + attributes=attributes, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ) as span: + response_future = None + start_time = 0 try: - yield from invoker(request_or_iterator, metadata) - except grpc.RpcError as err: - span.set_status(Status(StatusCode.ERROR)) - span.set_attribute( - SpanAttributes.RPC_GRPC_STATUS_CODE, err.code().value[0] + inject(mutable_metadata, setter=_carrier_setter) + metadata = tuple(mutable_metadata.items()) + new_client_call_details = _ClientCallDetails( + method=client_call_details.method, + timeout=client_call_details.timeout, + metadata=metadata, + credentials=client_call_details.credentials, + wait_for_ready=client_call_details.wait_for_ready, + compression=client_call_details.compression + ) + + start_time = self._start_duration_measurement() + request_iterator = self._record_streaming_request( + span, request_iterator, MessageTypeValues.SENT, attributes + ) + + response_future = continuation( + new_client_call_details, + request_iterator ) - raise err - def intercept_stream( - self, request_or_iterator, metadata, client_info, invoker + finally: + if not response_future: + span.end() + else: + response_future.add_done_callback( + _make_future_done_callback( + span, + self, + attributes, + start_time + ) + ) + + return response_future + + def intercept_stream_stream( + self, + continuation, + client_call_details: grpc.ClientCallDetails, + request_iterator: Iterator[ProtoMessage] ): if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return invoker(request_or_iterator, metadata) + return continuation(client_call_details, request_iterator) + + attributes = self._create_attributes(client_call_details.method) + + if not client_call_details.metadata: + mutable_metadata = OrderedDict() + else: + mutable_metadata = OrderedDict(client_call_details.metadata) + + with self._tracer.start_as_current_span( + name=client_call_details.method, + kind=trace.SpanKind.CLIENT, + attributes=attributes, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ) as span: + inject(mutable_metadata, setter=_carrier_setter) + metadata = tuple(mutable_metadata.items()) + new_client_call_details = _ClientCallDetails( + method=client_call_details.method, + timeout=client_call_details.timeout, + metadata=metadata, + credentials=client_call_details.credentials, + wait_for_ready=client_call_details.wait_for_ready, + compression=client_call_details.compression + ) - if client_info.is_server_stream: - return self._intercept_server_stream( - request_or_iterator, metadata, client_info, invoker + start_time = self._start_duration_measurement() + request_iterator = self._record_streaming_request( + span, request_iterator, MessageTypeValues.SENT, attributes ) - return self._intercept( - request_or_iterator, metadata, client_info, invoker - ) + response_iterator = continuation( + new_client_call_details, request_iterator + ) + + return _OpentelemetryResponseIterator( + response_iterator, + self, span, + attributes, + start_time + ) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py index 3c63f3f2b8..977e99f038 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/_utilities.py @@ -18,14 +18,15 @@ from contextlib import contextmanager from enum import Enum from timeit import default_timer +from types import TracebackType from typing import Callable, Dict, Generator, Iterable, Iterator, NoReturn, Optional import grpc from opentelemetry import metrics, trace -from opentelemetry.instrumentation.grpc._types import ProtoMessage +from opentelemetry.instrumentation.grpc._types import Metadata, ProtoMessage from opentelemetry.semconv.trace import MessageTypeValues, SpanAttributes from opentelemetry.trace.status import Status, StatusCode -from opentelemetry.util.types import Attributes, Metadata +from opentelemetry.util.types import Attributes _MESSAGE: str = "message" @@ -408,6 +409,147 @@ def _record_duration_manager( self._duration_histogram.record(duration, metric_attributes) +class _OpentelemetryResponseIterator( + grpc.Call, grpc.Future, grpc.RpcError, Iterator +): + + def __init__( + self, + response_iterator, + metric_recorder: _EventMetricRecorder, + span: trace.Span, + attributes: Attributes, + start_time: float + ) -> None: + self._iterator = response_iterator + self._metric_recorder = metric_recorder + self._span = span + self._attributes = attributes + self._start_time = start_time + self._response_id = 0 + + def __repr__(self): + return self._iterator.__repr__() + + def __str__(self): + return self._iterator.__str__() + + # Interface of grpc.RpcContext + + def add_callback(self, callback: Callable[[], None]) -> None: + return self._iterator.add_callback(callback) + + def cancel(self) -> bool: + return self._iterator.cancel() + + def is_active(self) -> bool: + return self._iterator.is_active() + + def time_remaining(self) -> Optional[float]: + return self._iterator.time_remaining() + + # Interface of grpc.Call + + def code(self) -> grpc.StatusCode: + return self._iterator.code() + + def details(self) -> Optional[str]: + return self._iterator.details() + + def initial_metadata(self) -> Metadata: + return self._iterator.initial_metadata() + + def trailing_metadata(self) -> Metadata: + return self._iterator.trailing_metadata() + + # Interface of grpc.Future + + # pylint: disable=invalid-name + def add_done_callback(self, fn: Callable[[grpc.Future], None]) -> None: + return self._iterator.add_done_callback(fn) + + def cancelled(self) -> bool: + return self._iterator.cancelled() + + def done(self) -> bool: + return self._iterator.done() + + def exception(self, timeout: Optional[float] = None) -> Exception: + return self._iterator.exception(timeout) + + def result(self, timeout: Optional[float] = None) -> ProtoMessage: + return self._iterator.result(timeout) + + def running(self) -> bool: + return self._iterator.running() + + def traceback(self, timeout: Optional[float] = None) -> TracebackType: + return self._iterator.traceback(timeout) + + # Iterator inteface + + def __iter__(self): + return self + + def next(self): + return self._next() + + def __next__(self): + return self._next() + + def _next(self) -> ProtoMessage: + with trace.use_span( + self._span, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False + ): + try: + response = self._iterator._next() + self._response_id += 1 + self._metric_recorder._record_unary_or_streaming_response( + self._span, + response, + MessageTypeValues.RECEIVED, + self._attributes, + response_id=self._response_id + ) + return response + except grpc.RpcError as exc: + code = exc.code() + details = exc.details() + self._span.set_attribute( + SpanAttributes.RPC_GRPC_STATUS_CODE, code.value[0] + ) + self._span.set_status( + Status( + status_code=StatusCode.ERROR, + description=f"{code}: {details}", + ) + ) + self._span.record_exception(exc) + + self._metric_recorder._record_num_of_responses_per_rpc( + self._response_id, self._attributes + ) + self._metric_recorder._record_duration( + self._start_time, self._attributes, self._iterator + ) + self._span.end() + raise + + except StopIteration: + self._metric_recorder._record_num_of_responses_per_rpc( + self._response_id, self._attributes + ) + + self._metric_recorder._record_duration( + self._start_time, self._attributes, self._iterator + ) + self._span.end() + raise + + # pylint:disable=abstract-method class _OpenTelemetryServicerContext(grpc.ServicerContext): diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/__init__.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/__init__.py deleted file mode 100644 index d5e2549bab..0000000000 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint:disable=import-outside-toplevel -# pylint:disable=import-self -# pylint:disable=no-name-in-module - -import abc - - -class UnaryClientInfo(abc.ABC): - """Consists of various information about a unary RPC on the - invocation-side. - - Attributes: - full_method: A string of the full RPC method, i.e., - /package.service/method. - timeout: The length of time in seconds to wait for the computation to - terminate or be cancelled, or None if this method should block until - the computation is terminated or is cancelled no matter how long that - takes. - """ - - -class StreamClientInfo(abc.ABC): - """Consists of various information about a stream RPC on the - invocation-side. - - Attributes: - full_method: A string of the full RPC method, i.e., - /package.service/method. - is_client_stream: Indicates whether the RPC is client-streaming. - is_server_stream: Indicates whether the RPC is server-streaming. - timeout: The length of time in seconds to wait for the computation to - terminate or be cancelled, or None if this method should block until - the computation is terminated or is cancelled no matter how long that - takes. - """ - - -class UnaryClientInterceptor(abc.ABC): - """Affords intercepting unary-unary RPCs on the invocation-side.""" - - @abc.abstractmethod - def intercept_unary(self, request, metadata, client_info, invoker): - """Intercepts unary-unary RPCs on the invocation-side. - - Args: - request: The request value for the RPC. - metadata: Optional :term:`metadata` to be transmitted to the - service-side of the RPC. - client_info: A UnaryClientInfo containing various information about - the RPC. - invoker: The handler to complete the RPC on the client. It is the - interceptor's responsibility to call it. - - Returns: - The result from calling invoker(request, metadata). - """ - raise NotImplementedError() - - -class StreamClientInterceptor(abc.ABC): - """Affords intercepting stream RPCs on the invocation-side.""" - - @abc.abstractmethod - def intercept_stream( - self, request_or_iterator, metadata, client_info, invoker - ): - """Intercepts stream RPCs on the invocation-side. - - Args: - request_or_iterator: The request value for the RPC if - `client_info.is_client_stream` is `false`; otherwise, an iterator of - request values. - metadata: Optional :term:`metadata` to be transmitted to the service-side - of the RPC. - client_info: A StreamClientInfo containing various information about - the RPC. - invoker: The handler to complete the RPC on the client. It is the - interceptor's responsibility to call it. - - Returns: - The result from calling invoker(metadata). - """ - raise NotImplementedError() - - -def intercept_channel(channel, *interceptors): - """Creates an intercepted channel. - - Args: - channel: A Channel. - interceptors: Zero or more UnaryClientInterceptors or - StreamClientInterceptors - - Returns: - A Channel. - - Raises: - TypeError: If an interceptor derives from neither UnaryClientInterceptor - nor StreamClientInterceptor. - """ - from . import _interceptor - - return _interceptor.intercept_channel(channel, *interceptors) - - -__all__ = ( - "UnaryClientInterceptor", - "StreamClientInfo", - "StreamClientInterceptor", - "intercept_channel", -) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/_interceptor.py b/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/_interceptor.py deleted file mode 100644 index 53ee46a20d..0000000000 --- a/instrumentation/opentelemetry-instrumentation-grpc/src/opentelemetry/instrumentation/grpc/grpcext/_interceptor.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint:disable=relative-beyond-top-level -# pylint:disable=no-member - -"""Implementation of gRPC Python interceptors.""" - - -import collections - -import grpc - -from opentelemetry.instrumentation.grpc import grpcext - - -class _UnaryClientInfo( - collections.namedtuple("_UnaryClientInfo", ("full_method", "timeout")) -): - pass - - -class _StreamClientInfo( - collections.namedtuple( - "_StreamClientInfo", - ("full_method", "is_client_stream", "is_server_stream", "timeout"), - ) -): - pass - - -class _InterceptorUnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): - def __init__(self, method, base_callable, interceptor): - self._method = method - self._base_callable = base_callable - self._interceptor = interceptor - - def __call__( - self, - request, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request, metadata): - return self._base_callable( - request, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _UnaryClientInfo(self._method, timeout) - return self._interceptor.intercept_unary( - request, metadata, client_info, invoker - ) - - def with_call( - self, - request, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request, metadata): - return self._base_callable.with_call( - request, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _UnaryClientInfo(self._method, timeout) - return self._interceptor.intercept_unary( - request, metadata, client_info, invoker - ) - - def future( - self, - request, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request, metadata): - return self._base_callable.future( - request, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _UnaryClientInfo(self._method, timeout) - return self._interceptor.intercept_unary( - request, metadata, client_info, invoker - ) - - -class _InterceptorUnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): - def __init__(self, method, base_callable, interceptor): - self._method = method - self._base_callable = base_callable - self._interceptor = interceptor - - def __call__( - self, - request, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request, metadata): - return self._base_callable( - request, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _StreamClientInfo(self._method, False, True, timeout) - return self._interceptor.intercept_stream( - request, metadata, client_info, invoker - ) - - -class _InterceptorStreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): - def __init__(self, method, base_callable, interceptor): - self._method = method - self._base_callable = base_callable - self._interceptor = interceptor - - def __call__( - self, - request_iterator, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request_iterator, metadata): - return self._base_callable( - request_iterator, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _StreamClientInfo(self._method, True, False, timeout) - return self._interceptor.intercept_stream( - request_iterator, metadata, client_info, invoker - ) - - def with_call( - self, - request_iterator, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request_iterator, metadata): - return self._base_callable.with_call( - request_iterator, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _StreamClientInfo(self._method, True, False, timeout) - return self._interceptor.intercept_stream( - request_iterator, metadata, client_info, invoker - ) - - def future( - self, - request_iterator, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request_iterator, metadata): - return self._base_callable.future( - request_iterator, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _StreamClientInfo(self._method, True, False, timeout) - return self._interceptor.intercept_stream( - request_iterator, metadata, client_info, invoker - ) - - -class _InterceptorStreamStreamMultiCallable(grpc.StreamStreamMultiCallable): - def __init__(self, method, base_callable, interceptor): - self._method = method - self._base_callable = base_callable - self._interceptor = interceptor - - def __call__( - self, - request_iterator, - timeout=None, - metadata=None, - credentials=None, - wait_for_ready=None, - compression=None, - ): - def invoker(request_iterator, metadata): - return self._base_callable( - request_iterator, - timeout, - metadata, - credentials, - wait_for_ready, - compression, - ) - - client_info = _StreamClientInfo(self._method, True, True, timeout) - return self._interceptor.intercept_stream( - request_iterator, metadata, client_info, invoker - ) - - -class _InterceptorChannel(grpc.Channel): - def __init__(self, channel, interceptor): - self._channel = channel - self._interceptor = interceptor - - def subscribe(self, *args, **kwargs): - self._channel.subscribe(*args, **kwargs) - - def unsubscribe(self, *args, **kwargs): - self._channel.unsubscribe(*args, **kwargs) - - def unary_unary( - self, method, request_serializer=None, response_deserializer=None - ): - base_callable = self._channel.unary_unary( - method, request_serializer, response_deserializer - ) - if isinstance(self._interceptor, grpcext.UnaryClientInterceptor): - return _InterceptorUnaryUnaryMultiCallable( - method, base_callable, self._interceptor - ) - return base_callable - - def unary_stream( - self, method, request_serializer=None, response_deserializer=None - ): - base_callable = self._channel.unary_stream( - method, request_serializer, response_deserializer - ) - if isinstance(self._interceptor, grpcext.StreamClientInterceptor): - return _InterceptorUnaryStreamMultiCallable( - method, base_callable, self._interceptor - ) - return base_callable - - def stream_unary( - self, method, request_serializer=None, response_deserializer=None - ): - base_callable = self._channel.stream_unary( - method, request_serializer, response_deserializer - ) - if isinstance(self._interceptor, grpcext.StreamClientInterceptor): - return _InterceptorStreamUnaryMultiCallable( - method, base_callable, self._interceptor - ) - return base_callable - - def stream_stream( - self, method, request_serializer=None, response_deserializer=None - ): - base_callable = self._channel.stream_stream( - method, request_serializer, response_deserializer - ) - if isinstance(self._interceptor, grpcext.StreamClientInterceptor): - return _InterceptorStreamStreamMultiCallable( - method, base_callable, self._interceptor - ) - return base_callable - - def close(self): - if not hasattr(self._channel, "close"): - raise RuntimeError( - "close() is not supported with the installed version of grpcio" - ) - self._channel.close() - - def __enter__(self): - """Enters the runtime context related to the channel object.""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Exits the runtime context related to the channel object.""" - self.close() - - -def intercept_channel(channel, *interceptors): - result = channel - for interceptor in interceptors: - if not isinstance( - interceptor, grpcext.UnaryClientInterceptor - ) and not isinstance(interceptor, grpcext.StreamClientInterceptor): - raise TypeError( - "interceptor must be either a " - "grpcext.UnaryClientInterceptor or a " - "grpcext.StreamClientInterceptor" - ) - result = _InterceptorChannel(result, interceptor) - return result From 73be9441d07ad544aa4885ed15064f686eb1a473 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:55:34 +0200 Subject: [PATCH 14/17] addes support for different arguments in message to support different errors --- .../tests/_server.py | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py index f46cce722f..fc0443a554 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/_server.py @@ -40,31 +40,36 @@ def SimpleMethod(self, request, context): return test_server_pb2.Response() elif request.request_data == "exception": raise ValueError(request.request_data) - elif request.request_data == "sleep": - time.sleep(0.5) + elif "sleep" in request.request_data: + sleep = float(request.request_data.split(" ", 1)[1]) + time.sleep(sleep) return test_server_pb2.Response( server_id=SERVER_ID, response_data="data" ) def ClientStreamingMethod(self, request_iterator, context): - data = list(request_iterator) + request = next(request_iterator) - if data[0].request_data == "abort": + if request.request_data == "abort": context.abort( - grpc.StatusCode.FAILED_PRECONDITION, data[0].request_data + grpc.StatusCode.FAILED_PRECONDITION, request.request_data ) - elif data[0].request_data == "cancel": + elif request.request_data == "cancel": context.cancel() return test_server_pb2.Response() - elif data[0].request_data == "error": + elif request.request_data == "error": context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(data[0].request_data) + context.set_details(request.request_data) return test_server_pb2.Response() - elif data[0].request_data == "exception": - raise ValueError(data[0].request_data) - elif data[0].request_data == "sleep": - time.sleep(0.5) + elif request.request_data == "exception": + raise ValueError(request.request_data) + elif "sleep" in request.request_data: + sleep = float(request.request_data.split(" ", 1)[1]) + time.sleep(sleep) + + for _ in request_iterator: + pass return test_server_pb2.Response( server_id=SERVER_ID, response_data="data" @@ -89,8 +94,9 @@ def ServerStreamingMethod(self, request, context): return test_server_pb2.Response() elif request.request_data == "exception": raise ValueError(request.request_data) - elif request.request_data == "sleep": - time.sleep(0.5) + elif "sleep" in request.request_data: + sleep = float(request.request_data.split(" ", 1)[1]) + time.sleep(sleep) for _ in range(5): yield test_server_pb2.Response( @@ -98,27 +104,31 @@ def ServerStreamingMethod(self, request, context): ) def BidirectionalStreamingMethod(self, request_iterator, context): - data = list(request_iterator) + request = next(request_iterator) yield test_server_pb2.Response( server_id=SERVER_ID, response_data="data" ) - if data[0].request_data == "abort": + if request.request_data == "abort": context.abort( - grpc.StatusCode.FAILED_PRECONDITION, data[0].request_data + grpc.StatusCode.FAILED_PRECONDITION, request.request_data ) - elif data[0].request_data == "cancel": + elif request.request_data == "cancel": context.cancel() return - elif data[0].request_data == "error": + elif request.request_data == "error": context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(data[0].request_data) + context.set_details(request.request_data) return - elif data[0].request_data == "exception": - raise ValueError(data[0].request_data) - elif data[0].request_data == "sleep": - time.sleep(0.5) + elif request.request_data == "exception": + raise ValueError(request.request_data) + elif "sleep" in request.request_data: + sleep = float(request.request_data.split(" ", 1)[1]) + time.sleep(sleep) + + for _ in request_iterator: + pass for _ in range(5): yield test_server_pb2.Response( From 6ba2462632763e0f660e06b5db4abe9c28dc4bff Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:55:43 +0200 Subject: [PATCH 15/17] not needed anymore --- .../tests/_client.py | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 instrumentation/opentelemetry-instrumentation-grpc/tests/_client.py diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/_client.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/_client.py deleted file mode 100644 index 69222b37a4..0000000000 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/_client.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .protobuf.test_server_pb2 import Request - -CLIENT_ID = 1 - - -def simple_method(stub, error=False): - request = Request( - client_id=CLIENT_ID, request_data="error" if error else "data" - ) - stub.SimpleMethod(request) - - -def simple_method_future(stub, error=False): - request = Request( - client_id=CLIENT_ID, request_data="error" if error else "data" - ) - return stub.SimpleMethod.future(request) - - -def client_streaming_method(stub, error=False): - # create a generator - def request_messages(): - for _ in range(5): - request = Request( - client_id=CLIENT_ID, request_data="error" if error else "data" - ) - yield request - - stub.ClientStreamingMethod(request_messages()) - - -def server_streaming_method(stub, error=False): - request = Request( - client_id=CLIENT_ID, request_data="error" if error else "data" - ) - response_iterator = stub.ServerStreamingMethod(request) - list(response_iterator) - - -def bidirectional_streaming_method(stub, error=False): - def request_messages(): - for _ in range(5): - request = Request( - client_id=CLIENT_ID, request_data="error" if error else "data" - ) - yield request - - response_iterator = stub.BidirectionalStreamingMethod(request_messages()) - - list(response_iterator) From bcb4e6c97a7bc1ce93942d6097fed03904efe71c Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:56:15 +0200 Subject: [PATCH 16/17] data point seen not logical --- .../tests/test_server_interceptor.py | 48 +++---------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py index 2d06cb92fd..c26a169f09 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_server_interceptor.py @@ -15,14 +15,12 @@ # pylint:disable=unused-argument # pylint:disable=no-self-use -from ftplib import error_perm -import logging import threading from concurrent import futures import grpc -from opentelemetry import trace, metrics +from opentelemetry import trace import opentelemetry.instrumentation.grpc from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer, server_interceptor from opentelemetry.sdk.metrics.export import Histogram, HistogramDataPoint @@ -370,9 +368,6 @@ def SimpleMethod(self, request, context): {} ) - logging.error("%r", self.memory_metrics_reader) - logging.error("%r", self.memory_metrics_reader.get_metrics_data()) - def test_create_span_streaming(self): """Check that the interceptor wraps calls with spans server-side, on a streaming call.""" @@ -1514,7 +1509,6 @@ def test_metrics(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -1581,10 +1575,6 @@ def test_metrics(self): }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) - def test_metrics_error(self): error_message = "error" @@ -1614,7 +1604,6 @@ def test_metrics_error(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -1682,9 +1671,6 @@ def test_metrics_error(self): ] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def test_metrics_three_calls(self): no_calls = 3 @@ -1714,7 +1700,6 @@ def test_metrics_three_calls(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -1784,9 +1769,6 @@ def test_metrics_three_calls(self): grpc.StatusCode.OK.value[0] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def test_metrics_client_streaming(self): @@ -1815,7 +1797,6 @@ def test_metrics_client_streaming(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -1888,9 +1869,6 @@ def test_metrics_client_streaming(self): grpc.StatusCode.OK.value[0] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def test_metrics_client_streaming_abort(self): @@ -1922,7 +1900,6 @@ def test_metrics_client_streaming_abort(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -1961,19 +1938,17 @@ def test_metrics_client_streaming_abort(self): self.assertEqual(point.count, 1) self.assertGreaterEqual(point.sum, 0) elif metric.name == "rpc.server.request.size": - self.assertEqual(point.count, len(requests)) - self.assertEqual( - point.sum, sum(map(len, requests)) - ) + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests[0])) elif metric.name == "rpc.server.response.size": self.assertEqual(point.count, 0) - # self.assertEqual(point.sum, 0) + self.assertEqual(point.sum, 0) elif metric.name == "rpc.server.requests_per_rpc": self.assertEqual(point.count, 1) - self.assertEqual(point.sum, len(requests)) + self.assertEqual(point.sum, 1) elif metric.name == "rpc.server.responses_per_rpc": self.assertEqual(point.count, 0) - # self.assertEqual(point.sum, 1) + self.assertEqual(point.sum, 0) self.assertMetricDataPointHasAttributes( point, @@ -1997,9 +1972,6 @@ def test_metrics_client_streaming_abort(self): ] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def test_metrics_server_streaming(self): @@ -2028,7 +2000,6 @@ def test_metrics_server_streaming(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -2101,9 +2072,6 @@ def test_metrics_server_streaming(self): grpc.StatusCode.OK.value[0] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def test_metrics_bidirectional_streaming(self): @@ -2134,7 +2102,6 @@ def test_metrics_bidirectional_streaming(self): server.stop(None) metrics_list = self.memory_metrics_reader.get_metrics_data() - data_point_seen = True self.assertNotEqual(len(metrics_list.resource_metrics), 0) for resource_metric in metrics_list.resource_metrics: @@ -2209,9 +2176,6 @@ def test_metrics_bidirectional_streaming(self): grpc.StatusCode.OK.value[0] }, ) - data_point_seen &= True - - self.assertTrue(data_point_seen) def get_latch(num): From 5f05d604a328a64f0dd17ad92c87898d92eb3234 Mon Sep 17 00:00:00 2001 From: Corvin Lasogga Date: Sun, 14 Aug 2022 12:56:34 +0200 Subject: [PATCH 17/17] complete test redesign of client interceptor --- .../tests/test_client_interceptor.py | 3288 ++++++++++++++++- 1 file changed, 3130 insertions(+), 158 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_client_interceptor.py b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_client_interceptor.py index 810ee930dd..66712f5d03 100644 --- a/instrumentation/opentelemetry-instrumentation-grpc/tests/test_client_interceptor.py +++ b/instrumentation/opentelemetry-instrumentation-grpc/tests/test_client_interceptor.py @@ -12,38 +12,65 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time +from typing import Iterator + import grpc from tests.protobuf import ( # pylint: disable=no-name-in-module test_server_pb2_grpc, ) import opentelemetry.instrumentation.grpc -from opentelemetry import context, trace +from opentelemetry import context, metrics, trace from opentelemetry.instrumentation.grpc import GrpcInstrumentorClient -from opentelemetry.instrumentation.grpc._client import ( - OpenTelemetryClientInterceptor, -) -from opentelemetry.instrumentation.grpc.grpcext._interceptor import ( - _UnaryClientInfo, -) +from opentelemetry.instrumentation.grpc._client import OpenTelemetryClientInterceptor +from opentelemetry.instrumentation.grpc._utilities import _ClientCallDetails from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY from opentelemetry.propagate import get_global_textmap, set_global_textmap +from opentelemetry.sdk.metrics.export import Histogram, HistogramDataPoint from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.test.mock_textmap import MockTextMapPropagator from opentelemetry.test.test_base import TestBase -from ._client import ( - bidirectional_streaming_method, - client_streaming_method, - server_streaming_method, - simple_method, - simple_method_future, -) from ._server import create_test_server -from .protobuf.test_server_pb2 import Request +from .protobuf.test_server_pb2 import Request, Response -# User defined interceptor. Is used in the tests along with the opentelemetry client interceptor. +CLIENT_ID = 1 + +_expected_metric_names = { + "rpc.client.duration": ( + Histogram, + "ms", + "Measures duration of RPC" + ), + "rpc.client.request.size": ( + Histogram, + "By", + "Measures size of RPC request messages (uncompressed)", + ), + "rpc.client.response.size": ( + Histogram, + "By", + "Measures size of RPC response messages (uncompressed)", + ), + "rpc.client.requests_per_rpc": ( + Histogram, + "1", + "Measures the number of messages received per RPC. " + "Should be 1 for all non-streaming RPCs" + ), + "rpc.client.responses_per_rpc": ( + Histogram, + "1", + "Measures the number of messages sent per RPC. " + "Should be 1 for all non-streaming RPCs", + ), +} + + +# User defined interceptor. Is used in the tests along with the opentelemetry +# client interceptor. class Interceptor( grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor, @@ -84,13 +111,35 @@ def _intercept_call( return continuation(client_call_details, request_or_iterator) -class TestClientProto(TestBase): +class TestOpenTelemetryClientInterceptor(TestBase): + + def assertEvent(self, event, name, attributes, msg=None): + self.assertEqual(event.name, name, msg=msg) + for key, val in attributes.items(): + out_msg = ( + str(event.attributes) + if msg is None else + ", ".join([msg, str(event.attributes)]) + ) + self.assertIn(key, event.attributes, msg=out_msg) + self.assertEqual(val, event.attributes[key], msg=out_msg) + + def assertEqualMetricInstrumentationScope(self, scope_metrics, module): + self.assertEqual(scope_metrics.scope.name, module.__name__) + self.assertEqual(scope_metrics.scope.version, module.__version__) + + def assertMetricDataPointHasAttributes(self, data_point, attributes): + for key, val in attributes.items(): + self.assertIn(key, data_point.attributes) + self.assertEqual(val, data_point.attributes[key]) + def setUp(self): super().setUp() GrpcInstrumentorClient().instrument() self.server = create_test_server(25565) self.server.start() - # use a user defined interceptor along with the opentelemetry client interceptor + # use a user defined interceptor along with the opentelemetry client + # interceptor interceptors = [Interceptor()] self.channel = grpc.insecure_channel("localhost:25565") self.channel = grpc.intercept_channel(self.channel, *interceptors) @@ -102,249 +151,3172 @@ def tearDown(self): self.server.stop(None) self.channel.close() - def test_unary_unary_future(self): - simple_method_future(self._stub).result() - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] + # + # Unary-Unary-RPC + # + + def test_unary_unary(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + response = self._stub.SimpleMethod(request) + + # check response + self.assertIsInstance(response, Response) + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") self.assertIs(span.kind, trace.SpanKind.CLIENT) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) - def test_unary_unary(self): - simple_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: response.ByteSize() + } + ) + + def test_unary_unary_abort(self): + data = "abort" + request = Request(client_id=CLIENT_ID, request_data=data) + + with self.assertRaises(grpc.RpcError): + self._stub.SimpleMethod(request) + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") self.assertIs(span.kind, trace.SpanKind.CLIENT) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) + # check span attributes self.assertSpanHasAttributes( span, { SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0], }, ) - def test_unary_stream(self): - server_streaming_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {data}", + ) - self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_unary_cancel_on_server_side(self): + data = "cancel" + request = Request(client_id=CLIENT_ID, request_data=data) + + with self.assertRaises(grpc.RpcError): + self._stub.SimpleMethod(request) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") self.assertIs(span.kind, trace.SpanKind.CLIENT) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) + # check span attributes self.assertSpanHasAttributes( span, { - SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], }, ) - def test_stream_unary(self): - client_streaming_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: " + f"{grpc.StatusCode.CANCELLED.value[1].upper()}", + ) - self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_unary_error(self): + data = "error" + request = Request(client_id=CLIENT_ID, request_data=data) + + with self.assertRaises(grpc.RpcError): + self._stub.SimpleMethod(request) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") self.assertIs(span.kind, trace.SpanKind.CLIENT) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) + # check span attributes self.assertSpanHasAttributes( span, { - SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0], }, ) - def test_stream_stream(self): - bidirectional_streaming_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) + + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_unary_exception(self): + data = "exception" + request = Request(client_id=CLIENT_ID, request_data=data) + + with self.assertRaises(grpc.RpcError): + self._stub.SimpleMethod(request) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0], + }, + ) + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) self.assertEqual( - span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + span.status.description, + f"{grpc.StatusCode.UNKNOWN}: " + f"Exception calling application: {data}", + ) + + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } ) + + def test_unary_unary_future(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + + response_future = self._stub.SimpleMethod.future(request) + response = response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + self.assertIsInstance(response, Response) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") self.assertIs(span.kind, trace.SpanKind.CLIENT) - # Check version and name in span's instrumentation info + # check version and name in span's instrumentation info self.assertEqualSpanInstrumentationInfo( span, opentelemetry.instrumentation.grpc ) + # check span attributes self.assertSpanHasAttributes( span, { - SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_METHOD: "SimpleMethod", SpanAttributes.RPC_SERVICE: "GRPCTestServer", SpanAttributes.RPC_SYSTEM: "grpc", - SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[ - 0 - ], + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] }, ) - def test_error_simple(self): - with self.assertRaises(grpc.RpcError): - simple_method(self._stub, error=True) + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: response.ByteSize() + } + ) + + def test_unary_unary_future_cancel_on_client_side(self): + sleep = 1.0 + data = f"sleep {sleep:f}" + request = Request(client_id=CLIENT_ID, request_data=data) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] - self.assertIs( - span.status.status_code, - trace.StatusCode.ERROR, + with self.assertRaises(grpc.FutureCancelledError): + response_future = self._stub.SimpleMethod.future(request) + time.sleep(sleep / 2) + response_future.cancel() + response_future.result() + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc ) - def test_error_stream_unary(self): - with self.assertRaises(grpc.RpcError): - client_streaming_method(self._stub, error=True) + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: Locally cancelled by application!", + ) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] - self.assertIs( - span.status.status_code, - trace.StatusCode.ERROR, + # check events + self.assertEqual(len(span.events), 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } ) - def test_error_unary_stream(self): + def test_unary_unary_future_error(self): + data = "error" + request = Request(client_id=CLIENT_ID, request_data=data) + with self.assertRaises(grpc.RpcError): - server_streaming_method(self._stub, error=True) + response_future = self._stub.SimpleMethod.future(request) + response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] - self.assertIs( - span.status.status_code, - trace.StatusCode.ERROR, + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + + self.assertEqual(span.name, "/GRPCTestServer/SimpleMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc ) - def test_error_stream_stream(self): - with self.assertRaises(grpc.RpcError): - bidirectional_streaming_method(self._stub, error=True) + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) - spans = self.memory_exporter.get_finished_spans() - self.assertEqual(len(spans), 1) - span = spans[0] - self.assertIs( - span.status.status_code, - trace.StatusCode.ERROR, + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0] + }, ) - def test_client_interceptor_trace_context_propagation( - self, - ): # pylint: disable=no-self-use - """ensure that client interceptor correctly inject trace context into all outgoing requests.""" - previous_propagator = get_global_textmap() - try: - set_global_textmap(MockTextMapPropagator()) - interceptor = OpenTelemetryClientInterceptor(trace.NoOpTracer()) + # check events + self.assertEqual(len(span.events), 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + self.assertEvent( + span.events[1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) - carrier = tuple() + def test_unary_unary_metrics(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + response = self._stub.SimpleMethod(request) - def invoker(request, metadata): - nonlocal carrier - carrier = metadata - return {} + # check response + self.assertIsInstance(response, Response) - request = Request(client_id=1, request_data="data") - interceptor.intercept_unary( - request, - {}, - _UnaryClientInfo( - full_method="/GRPCTestServer/SimpleMethod", timeout=None - ), - invoker=invoker, - ) + metrics_list = self.memory_metrics_reader.get_metrics_data() - assert len(carrier) == 2 - assert carrier[0][0] == "mock-traceid" - assert carrier[0][1] == "0" - assert carrier[1][0] == "mock-spanid" - assert carrier[1][1] == "0" + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) - finally: - set_global_textmap(previous_propagator) + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, 1) + if metric.name == "rpc.client.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.sum, request.ByteSize()) + elif metric.name == "rpc.client.response.size": + self.assertEqual( + point.sum, response.ByteSize() + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + def test_unary_unary_metrics_error(self): + data = "error" + request = Request(client_id=CLIENT_ID, request_data=data) + with self.assertRaises(grpc.RpcError): + self._stub.SimpleMethod(request) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) - 2 + ) + self.assertNotIn( + "rpc.client.response.size", scope_metric.metrics + ) + self.assertNotIn( + "rpc.client.responses_per_rpc", scope_metric.metrics + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, 1) + if metric.name == "rpc.client.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.sum, request.ByteSize()) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[ + 0 + ] + }, + ) + + def test_unary_unary_metrics_future(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + + response_future = self._stub.SimpleMethod.future(request) + response = response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + self.assertIsInstance(response, Response) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + self.assertEqual(point.count, 1) + if metric.name == "rpc.client.duration": + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.sum, request.ByteSize()) + elif metric.name == "rpc.client.response.size": + self.assertEqual( + point.sum, response.ByteSize() + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: "SimpleMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) def test_unary_unary_with_suppress_key(self): token = context.attach( context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) ) - try: - simple_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - finally: - context.detach(token) - self.assertEqual(len(spans), 0) - def test_unary_stream_with_suppress_key(self): - token = context.attach( - context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + request = Request( + client_id=CLIENT_ID, request_data="data" ) try: - server_streaming_method(self._stub) + self._stub.SimpleMethod(request) spans = self.memory_exporter.get_finished_spans() finally: context.detach(token) self.assertEqual(len(spans), 0) - def test_stream_unary_with_suppress_key(self): - token = context.attach( - context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + # + # Unary-Stream-RPC + # + + def test_unary_stream(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + response_iterator = self._stub.ServerStreamingMethod(request) + responses = list(response_iterator) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc ) - try: - client_streaming_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - finally: - context.detach(token) - self.assertEqual(len(spans), 0) - def test_stream_stream_with_suppress_key(self): - token = context.attach( - context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, ) - try: - bidirectional_streaming_method(self._stub) - spans = self.memory_exporter.get_finished_spans() - finally: - context.detach(token) - self.assertEqual(len(spans), 0) + + # check events + self.assertEqual(len(span.events), len(responses) + 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + }, + msg=f"Response ID: {res_id:d}" + ) + + def test_unary_stream_abort(self): + data = "abort" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(responses) + 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + } + ) + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_stream_cancel_on_client_side(self): + sleep = 1.0 + data = f"sleep {sleep:f}" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + responses.append(next(response_iterator)) + response_iterator.cancel() + responses.append(next(response_iterator)) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: Locally cancelled by application!" + ) + + # check events + self.assertGreater(len(span.events), len(responses) + 1) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + } + ) + + def test_unary_stream_cancel_on_server_side(self): + data = "cancel" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: " + f"{grpc.StatusCode.CANCELLED.value[1].upper()}", + ) + + # check events + self.assertEqual(len(span.events), len(responses) + 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + } + ) + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_stream_error(self): + data = "error" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(responses) + 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + } + ) + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_stream_exception(self): + data = "exception" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ServerStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.UNKNOWN}: " + f"Exception iterating responses: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(responses) + 2) + self.assertEvent( + span.events[0], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: request.ByteSize() + } + ) + for res_id, response in enumerate(responses, start=1): + self.assertEvent( + span.events[res_id], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + } + ) + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_unary_stream_metrics(self): + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + response_iterator = self._stub.ServerStreamingMethod(request) + responses = list(response_iterator) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, request.ByteSize()) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in responses) + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + def test_unary_stream_metrics_error(self): + data = "error" + request = Request(client_id=CLIENT_ID, request_data=data) + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.ServerStreamingMethod(request) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, request.ByteSize()) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in responses) + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "ServerStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[ + 0 + ] + }, + ) + + def test_unary_stream_with_suppress_key(self): + token = context.attach( + context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + ) + data = "data" + request = Request(client_id=CLIENT_ID, request_data=data) + + try: + list(self._stub.ServerStreamingMethod(request)) + spans = self.memory_exporter.get_finished_spans() + finally: + context.detach(token) + self.assertEqual(len(spans), 0) + + # + # Stream-Unary-RPC + # + + def test_stream_unary(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response = self._stub.ClientStreamingMethod(request_messages(data)) + + # check response + self.assertIsInstance(response, Response) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + }, + ) + + def test_stream_unary_abort(self): + data = "abort" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + self._stub.ClientStreamingMethod(request_messages(data)) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_unary_cancel_on_server_side(self): + data = "cancel" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + self._stub.ClientStreamingMethod(request_messages(data)) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: " + f"{grpc.StatusCode.CANCELLED.value[1].upper()}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_unary_error(self): + data = "error" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + self._stub.ClientStreamingMethod(request_messages(data)) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_unary_exception(self): + data = "exception" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + self._stub.ClientStreamingMethod(request_messages(data)) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.UNKNOWN}: " + f"Exception calling application: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "exception", + { + "exception.type": "_InactiveRpcError", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_unary_future(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response_future = self._stub.ClientStreamingMethod.future( + request_messages(data) + ) + response = response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + self.assertIsInstance(response, Response) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: 1, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + response.ByteSize() + }, + ) + + def test_stream_unary_future_cancel_on_client_side(self): + sleep = 1.0 + data = f"sleep {sleep:f}" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.FutureCancelledError): + response_future = self._stub.ClientStreamingMethod.future( + request_messages(data) + ) + time.sleep(sleep / 2) + response_future.cancel() + response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: Locally cancelled by application!", + ) + + # check events + self.assertEqual(len(span.events), len(requests)) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + + def test_stream_unary_future_error(self): + data = "error" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + response_future = self._stub.ClientStreamingMethod.future( + request_messages(data) + ) + response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/ClientStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0] + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + 1) + for req_id, request in enumerate(requests, start=1): + self.assertEvent( + span.events[req_id - 1], + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + request.ByteSize() + }, + msg=f"Request ID: {req_id:d}" + ) + self.assertEvent( + span.events[len(requests)], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_unary_metrics(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response = self._stub.ClientStreamingMethod(request_messages(data)) + + # check response + self.assertIsInstance(response, Response) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in requests) + ) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, 1) + self.assertEqual( + point.sum, response.ByteSize() + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + def test_stream_unary_metrics_error(self): + data = "error" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + with self.assertRaises(grpc.RpcError): + self._stub.ClientStreamingMethod(request_messages(data)) + + # streams depend on threads -> wait for completition + time.sleep(0.1) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) - 2, + msg=", ".join([m.name for m in scope_metric.metrics]) + ) + self.assertNotIn( + "rpc.client.response.size", scope_metric.metrics + ) + self.assertNotIn( + "rpc.client.responses_per_rpc", scope_metric.metrics + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in requests) + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[ + 0 + ] + }, + ) + + def test_stream_unary_metrics_future(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response_future = self._stub.ClientStreamingMethod.future( + request_messages(data) + ) + response = response_future.result() + + # check API and response + self.assertIsInstance(response_future, grpc.Call) + self.assertIsInstance(response_future, grpc.Future) + self.assertIsInstance(response, Response) + + # span is finished in done callback which is run in different thread, + # so we are waiting for it + for _ in range(5): + if len(self.memory_exporter.get_finished_spans()) == 0: + time.sleep(0.1) + else: + break + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in requests) + ) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, 1) + self.assertEqual( + point.sum, response.ByteSize() + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, 1) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "ClientStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + def test_stream_unary_with_suppress_key(self): + token = context.attach( + context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + ) + + def request_messages(data): + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + yield request + + try: + self._stub.ClientStreamingMethod(request_messages("data")) + spans = self.memory_exporter.get_finished_spans() + finally: + context.detach(token) + self.assertEqual(len(spans), 0) + + # + # Stream-Stream-RPC + # + + def test_stream_stream(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + responses = list(response_iterator) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual( + span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + ) + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + # check events + self.assertEqual(len(span.events), len(requests) + len(responses)) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event in span.events: + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + def test_stream_stream_abort(self): + data = "abort" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual( + span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + ) + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.FAILED_PRECONDITION.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.FAILED_PRECONDITION}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + len(responses) + 1) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event in span.events[:-1]: + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_stream_cancel_on_client_side(self): + sleep = 1.0 + data = f"sleep {sleep:f}" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + responses.append(next(response_iterator)) + response_iterator.cancel() + responses.append(next(response_iterator)) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual( + span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + ) + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: Locally cancelled by application!" + ) + + # check events + self.assertGreaterEqual( + len(span.events), len(requests) + len(responses), + msg=", ".join([e.name for e in span.events]) + ) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event_id, event in enumerate(span.events, start=1): + if event.name == "message": + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + elif event.name == "exception": + self.assertEvent( + event, + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + self.assertEqual(event_id, len(span.events)) + else: + self.fail(f"unknown event name: {event.name}") + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + def test_stream_stream_cancel_on_server_side(self): + data = "cancel" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual( + span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + ) + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.CANCELLED.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.CANCELLED}: " + f"{grpc.StatusCode.CANCELLED.value[1].upper()}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + len(responses) + 1) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event in span.events[:-1]: + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_stream_error(self): + data = "error" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual( + span.name, "/GRPCTestServer/BidirectionalStreamingMethod" + ) + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.INVALID_ARGUMENT}: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + len(responses) + 1) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event in span.events[:-1]: + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_stream_exception(self): + data = "exception" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + spans_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans_list), 1) + span = spans_list[0] + self.assertEqual(span.name, "/GRPCTestServer/BidirectionalStreamingMethod") + self.assertIs(span.kind, trace.SpanKind.CLIENT) + + # check version and name in span's instrumentation info + self.assertEqualSpanInstrumentationInfo( + span, opentelemetry.instrumentation.grpc + ) + + # check span attributes + self.assertSpanHasAttributes( + span, + { + SpanAttributes.RPC_METHOD: "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.UNKNOWN.value[0], + }, + ) + + # make sure this span errored, with the right status and detail + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual( + span.status.description, + f"{grpc.StatusCode.UNKNOWN}: " + f"Exception iterating responses: {data}", + ) + + # check events + self.assertEqual(len(span.events), len(requests) + len(responses) + 1) + # requests and responses occur random + req_id = 0 + res_id = 0 + for event in span.events[:-1]: + self.assertIn(SpanAttributes.MESSAGE_TYPE, event.attributes) + if event.attributes[SpanAttributes.MESSAGE_TYPE] == "SENT": + req_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "SENT", + SpanAttributes.MESSAGE_ID: req_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + requests[req_id - 1].ByteSize() + } + ) + elif event.attributes[SpanAttributes.MESSAGE_TYPE] == "RECEIVED": + res_id += 1 + self.assertEvent( + event, + "message", + { + SpanAttributes.MESSAGE_TYPE: "RECEIVED", + SpanAttributes.MESSAGE_ID: res_id, + SpanAttributes.MESSAGE_UNCOMPRESSED_SIZE: + responses[res_id - 1].ByteSize() + } + ) + else: + self.fail( + "unknown message type: " + f"{event.attributes[SpanAttributes.MESSAGE_TYPE]}" + ) + self.assertEqual(len(requests), req_id) + self.assertEqual(len(responses), res_id) + + self.assertEvent( + span.events[-1], + "exception", + { + "exception.type": "_MultiThreadedRendezvous", + # "exception.message": error_message, + # "exception.stacktrace": "...", + "exception.escaped": str(False), + } + ) + + def test_stream_stream_metrics(self): + data = "data" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + responses = list(response_iterator) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in requests) + ) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in responses) + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.OK.value[0] + }, + ) + + def test_stream_stream_metrics_error(self): + data = "error" + requests = [] + + def request_messages(data): + nonlocal requests + + for _ in range(5): + request = Request(client_id=CLIENT_ID, request_data=data) + requests.append(request) + yield request + + responses = [] + with self.assertRaises(grpc.RpcError): + response_iterator = self._stub.BidirectionalStreamingMethod( + request_messages(data) + ) + for response in response_iterator: + responses.append(response) + + # check API and response + self.assertIsInstance(response_iterator, grpc.Call) + self.assertIsInstance(response_iterator, grpc.Future) + self.assertIsInstance(response_iterator, Iterator) + + # streams depend on threads -> wait for completition + time.sleep(0.1) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + + self.assertNotEqual(len(metrics_list.resource_metrics), 0) + for resource_metric in metrics_list.resource_metrics: + self.assertNotEqual(len(resource_metric.scope_metrics), 0) + for scope_metric in resource_metric.scope_metrics: + self.assertNotEqual(len(scope_metric.metrics), 0) + self.assertEqualMetricInstrumentationScope( + scope_metric, opentelemetry.instrumentation.grpc + ) + self.assertEqual( + len(scope_metric.metrics), len(_expected_metric_names), + msg=", ".join([m.name for m in scope_metric.metrics]) + ) + + for metric in scope_metric.metrics: + self.assertIn(metric.name, _expected_metric_names) + self.assertIsInstance( + metric.data, _expected_metric_names[metric.name][0] + ) + self.assertEqual( + metric.unit, _expected_metric_names[metric.name][1] + ) + self.assertEqual( + metric.description, + _expected_metric_names[metric.name][2] + ) + + data_points = list(metric.data.data_points) + self.assertEqual(len(data_points), 1) + + for point in data_points: + if isinstance(metric.data, Histogram): + self.assertIsInstance( + point, HistogramDataPoint + ) + if metric.name == "rpc.client.duration": + self.assertEqual(point.count, 1) + self.assertGreaterEqual(point.sum, 0) + elif metric.name == "rpc.client.request.size": + self.assertEqual(point.count, len(requests)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in requests) + ) + elif metric.name == "rpc.client.response.size": + self.assertEqual(point.count, len(responses)) + self.assertEqual( + point.sum, + sum(r.ByteSize() for r in responses) + ) + elif metric.name == "rpc.client.requests_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(requests)) + elif metric.name == "rpc.client.responses_per_rpc": + self.assertEqual(point.count, 1) + self.assertEqual(point.sum, len(responses)) + + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_METHOD: + "BidirectionalStreamingMethod", + SpanAttributes.RPC_SERVICE: "GRPCTestServer", + SpanAttributes.RPC_SYSTEM: "grpc", + }, + ) + if metric.name == "rpc.client.duration": + self.assertMetricDataPointHasAttributes( + point, + { + SpanAttributes.RPC_GRPC_STATUS_CODE: + grpc.StatusCode.INVALID_ARGUMENT.value[ + 0 + ] + }, + ) + + def test_stream_stream_with_suppress_key(self): + token = context.attach( + context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + ) + + def request_messages(data): + for _ in range(5): + yield Request(client_id=CLIENT_ID, request_data=data) + + try: + self._stub.BidirectionalStreamingMethod(request_messages("data")) + spans = self.memory_exporter.get_finished_spans() + finally: + context.detach(token) + + self.assertEqual(len(spans), 0) + + def test_client_interceptor_trace_context_propagation( + self, + ): # pylint: disable=no-self-use + """ensure that client interceptor correctly inject trace context into + all outgoing requests. + """ + + previous_propagator = get_global_textmap() + try: + set_global_textmap(MockTextMapPropagator()) + interceptor = OpenTelemetryClientInterceptor( + metrics.NoOpMeter("test"), trace.NoOpTracer() + ) + + carrier = tuple() + + def invoker(client_call_details, request): + nonlocal carrier + carrier = client_call_details.metadata + return {} + + request = Request(client_id=1, request_data="data") + interceptor.intercept_unary_unary( + invoker, + _ClientCallDetails( + method="/GRPCTestServer/SimpleMethod", + timeout=None, + metadata=None, + credentials=None, + wait_for_ready=False, + compression=None + ), + request + ) + + assert len(carrier) == 2 + assert carrier[0][0] == "mock-traceid" + assert carrier[0][1] == "0" + assert carrier[1][0] == "mock-spanid" + assert carrier[1][1] == "0" + + finally: + set_global_textmap(previous_propagator)