From 98ad35edfc0a0e7631aa47fbd0316e4575adb623 Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Thu, 7 Sep 2023 21:30:35 +0300 Subject: [PATCH 1/8] Distributed tracing support with Micrometer and OpenTelemetry --- grpc-spring-boot-starter/build.gradle | 2 + .../tracing/OtelConfiguration.java | 42 +++++++++++++++ .../tracing/TracingClientInterceptor.java | 33 ++++++++++++ .../tracing/TracingServerInterceptor.java | 52 +++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java create mode 100644 grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java create mode 100644 grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java diff --git a/grpc-spring-boot-starter/build.gradle b/grpc-spring-boot-starter/build.gradle index 187fb4d..4ba5e4e 100644 --- a/grpc-spring-boot-starter/build.gradle +++ b/grpc-spring-boot-starter/build.gradle @@ -286,6 +286,8 @@ dependencies { compileOnly "org.springframework.boot:spring-boot-starter-actuator" compileOnly "org.springframework.boot:spring-boot-starter-validation" compileOnly 'org.springframework.cloud:spring-cloud-starter-consul-discovery' + compileOnly 'io.opentelemetry:opentelemetry-exporter-otlp:1.29.0' + compileOnly 'io.micrometer:micrometer-tracing-bridge-otel:1.1.4' testImplementation "javax.annotation:javax.annotation-api:1.3.2" diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java new file mode 100644 index 0000000..ce8bd6c --- /dev/null +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java @@ -0,0 +1,42 @@ +package org.lognet.springboot.grpc.autoconfigure.tracing; + +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporterBuilder; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import org.lognet.springboot.grpc.tracing.TracingServerInterceptor; +import org.springframework.boot.actuate.autoconfigure.tracing.otlp.OtlpProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties(OtlpProperties.class) +public class OtelConfiguration { + + @Bean + public OtlpGrpcSpanExporter otlpExporter(OtlpProperties properties) { + OtlpGrpcSpanExporterBuilder builder = OtlpGrpcSpanExporter.builder().setEndpoint(properties.getEndpoint()); + return builder.build(); + } + + @Bean + public Tracer tracer(OtlpGrpcSpanExporter otlpExporter) { + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(otlpExporter).build()) + .build(); + + OpenTelemetrySdk openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .buildAndRegisterGlobal(); + + return openTelemetry.getTracerProvider().get("grpc-spring-boot-starter"); + } + + @Bean + public TracingServerInterceptor tracingServerInterceptor(Tracer tracer) { + return new TracingServerInterceptor(tracer); + } +} diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java new file mode 100644 index 0000000..8f62abb --- /dev/null +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java @@ -0,0 +1,33 @@ +package org.lognet.springboot.grpc.tracing; + +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ForwardingClientCall; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.TraceId; + +public class TracingClientInterceptor implements ClientInterceptor { + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + + return new ForwardingClientCall.SimpleForwardingClientCall(next.newCall(method, callOptions)) { + @Override + public void start(ClientCall.Listener responseListener, Metadata headers) { + Span currentSpan = Span.current(); + String traceId = currentSpan.getSpanContext().getTraceId(); + String spanId = currentSpan.getSpanContext().getSpanId(); + + if (!traceId.equals(TraceId.getInvalid())) { + headers.put(Metadata.Key.of("traceId", Metadata.ASCII_STRING_MARSHALLER), traceId); + headers.put(Metadata.Key.of("spanId", Metadata.ASCII_STRING_MARSHALLER), spanId); + } + super.start(responseListener, headers); + } + }; + } +} diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java new file mode 100644 index 0000000..40f010e --- /dev/null +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java @@ -0,0 +1,52 @@ +package org.lognet.springboot.grpc.tracing; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; + +public class TracingServerInterceptor implements ServerInterceptor { + + private final Tracer tracer; + + public TracingServerInterceptor(Tracer tracer) { + this.tracer = tracer; + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, Metadata headers, ServerCallHandler next) { + + String traceId = headers.get(Metadata.Key.of("traceId", Metadata.ASCII_STRING_MARSHALLER)); + String spanId = headers.get(Metadata.Key.of("spanId", Metadata.ASCII_STRING_MARSHALLER)); + + Context spanContext = createSpanContext(traceId, spanId); + SpanBuilder spanBuilder = tracer.spanBuilder("grpc-spring-boot-starter-span").setParent(spanContext); + Span span = spanBuilder.startSpan(); + Context scopedContext = Context.current().with(span); + + try (Scope scope = scopedContext.makeCurrent()) { + return next.startCall(call, headers); + } finally { + span.end(); + } + } + + private Context createSpanContext(String traceId, String spanId) { + SpanContext spanContext = SpanContext.createFromRemoteParent( + traceId != null ? traceId : "", + spanId != null ? spanId : "", + TraceFlags.getDefault(), + TraceState.getDefault() + ); + return Context.current().with(Span.wrap(spanContext)); + } +} From 01e76208f548342f6378101ab5d9e87ac251d6bf Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Thu, 7 Sep 2023 21:38:41 +0300 Subject: [PATCH 2/8] add readme --- README.adoc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.adoc b/README.adoc index 58fb064..3d81461 100644 --- a/README.adoc +++ b/README.adoc @@ -315,6 +315,27 @@ image:./images/interceptors_003.png[] This started is *natively* supported by `spring-cloud-sleuth` project. + Please continue to https://docs.spring.io/spring-cloud-sleuth/docs/current/reference/html/integrations.html#sleuth-rpc-grpc-integration[sleuth grpc integration]. +=== Distributed tracing support (OpenTelemetry) + +OpenTelemetry distributed tracing can enabled by including: +``` +implementation 'io.opentelemetry:opentelemetry-exporter-otlp:[version]' +implementation 'io.micrometer:micrometer-tracing-bridge-otel:[version]' +``` + +Enable tracing for client by adding `TracingClientInterceptor` to `ManagedChannelBuilder`: +``` +ManagedChannelBuilder.forAddress(host, port) + ... +.intercept(TracingClientInterceptor()) +.build() +``` + +Add tracing interceptor `TracingServerInterceptor` for your `GRpcService`: +``` +@GRpcService(interceptors = { TracingServerInterceptor.class }) +... +``` === GRPC server metrics (Micrometer.io integration) From af9e2896feeea2cc2cf030395e0767ac6bcfebfd Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Thu, 7 Sep 2023 21:41:10 +0300 Subject: [PATCH 3/8] add LogInterceptor --- README.adoc | 2 +- .../springboot/grpc/log/LogInterceptor.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java diff --git a/README.adoc b/README.adoc index 3d81461..84652d7 100644 --- a/README.adoc +++ b/README.adoc @@ -333,7 +333,7 @@ ManagedChannelBuilder.forAddress(host, port) Add tracing interceptor `TracingServerInterceptor` for your `GRpcService`: ``` -@GRpcService(interceptors = { TracingServerInterceptor.class }) +@GRpcService(interceptors = {LogInterceptor.class, TracingServerInterceptor.class }) ... ``` diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java new file mode 100644 index 0000000..8cd1937 --- /dev/null +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java @@ -0,0 +1,22 @@ +package org.lognet.springboot.grpc.log; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class LogInterceptor implements ServerInterceptor { + + private final Logger logger = LoggerFactory.getLogger(LogInterceptor.class); + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, Metadata headers, ServerCallHandler next) { + logger.info(call.getMethodDescriptor().getFullMethodName()); + return next.startCall(call, headers); + } +} From 8fae4d3988d252ff6b9bb1a3fd497bf7c077d55f Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Thu, 7 Sep 2023 21:53:10 +0300 Subject: [PATCH 4/8] readme update --- README.adoc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.adoc b/README.adoc index 84652d7..effff87 100644 --- a/README.adoc +++ b/README.adoc @@ -318,24 +318,26 @@ Please continue to https://docs.spring.io/spring-cloud-sleuth/docs/current/refer === Distributed tracing support (OpenTelemetry) OpenTelemetry distributed tracing can enabled by including: -``` +[source,groovy] +---- implementation 'io.opentelemetry:opentelemetry-exporter-otlp:[version]' implementation 'io.micrometer:micrometer-tracing-bridge-otel:[version]' -``` +---- Enable tracing for client by adding `TracingClientInterceptor` to `ManagedChannelBuilder`: -``` +[source,java] +---- ManagedChannelBuilder.forAddress(host, port) ... .intercept(TracingClientInterceptor()) .build() -``` +---- Add tracing interceptor `TracingServerInterceptor` for your `GRpcService`: -``` +[source,java] +---- @GRpcService(interceptors = {LogInterceptor.class, TracingServerInterceptor.class }) -... -``` +---- === GRPC server metrics (Micrometer.io integration) From 2a2f309bd00a029dcd5d929559ed511811acf3a9 Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Wed, 13 Sep 2023 13:36:13 +0300 Subject: [PATCH 5/8] move TracingClientInterceptor to grpc-client-spring-boot-starter --- grpc-client-spring-boot-starter/build.gradle | 2 ++ .../springboot/grpc/tracing/TracingClientInterceptor.java | 0 2 files changed, 2 insertions(+) rename {grpc-spring-boot-starter => grpc-client-spring-boot-starter}/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java (100%) diff --git a/grpc-client-spring-boot-starter/build.gradle b/grpc-client-spring-boot-starter/build.gradle index 33fbd86..33ddfa7 100644 --- a/grpc-client-spring-boot-starter/build.gradle +++ b/grpc-client-spring-boot-starter/build.gradle @@ -114,6 +114,8 @@ signing { dependencies { api "io.grpc:grpc-api:${grpcVersion}" + compileOnly 'io.opentelemetry:opentelemetry-exporter-otlp:1.29.0' + compileOnly 'io.micrometer:micrometer-tracing-bridge-otel:1.1.4' } compileJava.dependsOn(processResources) diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java b/grpc-client-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java similarity index 100% rename from grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java rename to grpc-client-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingClientInterceptor.java From 2d5d1b570460d16a3fc2e3ecda026e3b3f7f498c Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Wed, 13 Sep 2023 15:46:05 +0300 Subject: [PATCH 6/8] review --- grpc-client-spring-boot-starter/build.gradle | 1 - .../tracing/OtelConfiguration.java | 2 ++ .../springboot/grpc/log/LogInterceptor.java | 22 ------------------- 3 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java diff --git a/grpc-client-spring-boot-starter/build.gradle b/grpc-client-spring-boot-starter/build.gradle index 33ddfa7..212a1d1 100644 --- a/grpc-client-spring-boot-starter/build.gradle +++ b/grpc-client-spring-boot-starter/build.gradle @@ -115,7 +115,6 @@ signing { dependencies { api "io.grpc:grpc-api:${grpcVersion}" compileOnly 'io.opentelemetry:opentelemetry-exporter-otlp:1.29.0' - compileOnly 'io.micrometer:micrometer-tracing-bridge-otel:1.1.4' } compileJava.dependsOn(processResources) diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java index ce8bd6c..708dbea 100644 --- a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/autoconfigure/tracing/OtelConfiguration.java @@ -8,11 +8,13 @@ import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import org.lognet.springboot.grpc.tracing.TracingServerInterceptor; import org.springframework.boot.actuate.autoconfigure.tracing.otlp.OtlpProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration +@ConditionalOnClass({OpenTelemetrySdk.class}) @EnableConfigurationProperties(OtlpProperties.class) public class OtelConfiguration { diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java deleted file mode 100644 index 8cd1937..0000000 --- a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/log/LogInterceptor.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.lognet.springboot.grpc.log; - -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -@Component -public class LogInterceptor implements ServerInterceptor { - - private final Logger logger = LoggerFactory.getLogger(LogInterceptor.class); - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - logger.info(call.getMethodDescriptor().getFullMethodName()); - return next.startCall(call, headers); - } -} From fda47dc1659ae7d81d74d1bf8e584104d3049f1e Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Wed, 13 Sep 2023 15:48:34 +0300 Subject: [PATCH 7/8] add @Order(20) for TracingServerInterceptor --- .../springboot/grpc/tracing/TracingServerInterceptor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java index 40f010e..289cba1 100644 --- a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java @@ -12,7 +12,9 @@ import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; +import org.springframework.core.annotation.Order; +@Order(20) public class TracingServerInterceptor implements ServerInterceptor { private final Tracer tracer; From 56e34d918fc79aad2603d14762c90ea20f1e68a9 Mon Sep 17 00:00:00 2001 From: Oleksandr Shevchenko Date: Wed, 13 Sep 2023 16:26:50 +0300 Subject: [PATCH 8/8] set HIGHEST_PRECEDENCE --- .../tracing/TracingServerInterceptor.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java index 289cba1..3d6dddb 100644 --- a/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java +++ b/grpc-spring-boot-starter/src/main/java/org/lognet/springboot/grpc/tracing/TracingServerInterceptor.java @@ -4,18 +4,20 @@ import io.grpc.ServerCall; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.*; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; -import org.springframework.core.annotation.Order; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.springframework.core.Ordered; -@Order(20) -public class TracingServerInterceptor implements ServerInterceptor { +import java.util.Optional; + +public class TracingServerInterceptor implements ServerInterceptor, Ordered { + + @Setter + @Accessors(fluent = true) + private Integer order; private final Tracer tracer; @@ -51,4 +53,9 @@ private Context createSpanContext(String traceId, String spanId) { ); return Context.current().with(Span.wrap(spanContext)); } + + @Override + public int getOrder() { + return Optional.ofNullable(order).orElse(HIGHEST_PRECEDENCE); + } }