From d9aaf2d5870d33e3cc3d1f680e448ae90687f1c3 Mon Sep 17 00:00:00 2001 From: scaugrated <357379243@qq.com> Date: Mon, 3 Jul 2023 14:17:02 -0700 Subject: [PATCH] generate error/fault metrics by aws sdk status code (#924) --- .../awsxray/AwsSpanMetricsProcessor.java | 62 ++++++++++- .../awsxray/AwsSpanMetricsProcessorTest.java | 103 +++++++++++++++++- 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java index 4fec1686e..fb2c2a103 100644 --- a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java @@ -15,7 +15,11 @@ import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.EventData; import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData; +import java.lang.reflect.Method; +import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; /** @@ -92,7 +96,7 @@ public void onEnd(ReadableSpan span) { // Only record metrics if non-empty attributes are returned. if (!attributes.isEmpty()) { - recordErrorOrFault(span, attributes); + recordErrorOrFault(spanData, attributes); recordLatency(span, attributes); } } @@ -102,10 +106,14 @@ public boolean isEndRequired() { return true; } - private void recordErrorOrFault(ReadableSpan span, Attributes attributes) { - Long httpStatusCode = span.getAttribute(HTTP_STATUS_CODE); + private void recordErrorOrFault(SpanData spanData, Attributes attributes) { + Long httpStatusCode = spanData.getAttributes().get(HTTP_STATUS_CODE); if (httpStatusCode == null) { - return; + httpStatusCode = getAwsStatusCode(spanData); + + if (httpStatusCode == null || httpStatusCode < 100L || httpStatusCode > 599L) { + return; + } } if (httpStatusCode >= ERROR_CODE_LOWER_BOUND && httpStatusCode <= ERROR_CODE_UPPER_BOUND) { @@ -116,6 +124,52 @@ private void recordErrorOrFault(ReadableSpan span, Attributes attributes) { } } + /** + * Attempt to pull status code from spans produced by AWS SDK instrumentation (both v1 and v2). + * AWS SDK instrumentation does not populate http.status_code when non-200 status codes are + * returned, as the AWS SDK throws exceptions rather than returning responses with status codes. + * To work around this, we are attempting to get the exception out of the events, then calling + * getStatusCode (for AWS SDK V1) and statusCode (for AWS SDK V2) to get the status code fromt the + * exception. We rely on reflection here because we cannot cast the throwable to + * AmazonServiceExceptions (V1) or AwsServiceExceptions (V2) because the throwable comes from a + * separate class loader and attempts to cast will fail with ClassCastException. + * + *

TODO: Short term workaround. This can be completely removed once + * https://github.com/open-telemetry/opentelemetry-java-contrib/issues/919 is resolved. + */ + @Nullable + private static Long getAwsStatusCode(SpanData spanData) { + String scopeName = spanData.getInstrumentationScopeInfo().getName(); + if (!scopeName.contains("aws-sdk")) { + return null; + } + + for (EventData event : spanData.getEvents()) { + if (event instanceof ExceptionEventData) { + ExceptionEventData exceptionEvent = (ExceptionEventData) event; + Throwable throwable = exceptionEvent.getException(); + + try { + Method method = throwable.getClass().getMethod("getStatusCode", new Class[] {}); + Object code = method.invoke(throwable, new Object[] {}); + return Long.valueOf((Integer) code); + } catch (Exception e) { + // Take no action + } + + try { + Method method = throwable.getClass().getMethod("statusCode", new Class[] {}); + Object code = method.invoke(throwable, new Object[] {}); + return Long.valueOf((Integer) code); + } catch (Exception e) { + // Take no action + } + } + } + + return null; + } + private void recordLatency(ReadableSpan span, Attributes attributes) { long nanos = span.getLatencyNanos(); double millis = nanos / NANOS_TO_MILLIS; diff --git a/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java index 6f6fd2b97..c2109b267 100644 --- a/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java +++ b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java @@ -22,16 +22,21 @@ import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.data.EventData; import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** Unit tests for {@link AwsSpanMetricsProcessor}. */ class AwsSpanMetricsProcessorTest { - // Test constants private static final boolean CONTAINS_ATTRIBUTES = true; private static final boolean CONTAINS_NO_ATTRIBUTES = false; @@ -56,6 +61,32 @@ private enum ExpectedStatusMetric { private AwsSpanMetricsProcessor awsSpanMetricsProcessor; + static class ThrowableWithMethodGetStatusCode extends Throwable { + private final int httpStatusCode; + + ThrowableWithMethodGetStatusCode(int httpStatusCode) { + this.httpStatusCode = httpStatusCode; + } + + public int getStatusCode() { + return this.httpStatusCode; + } + } + + static class ThrowableWithMethodStatusCode extends Throwable { + private final int httpStatusCode; + + ThrowableWithMethodStatusCode(int httpStatusCode) { + this.httpStatusCode = httpStatusCode; + } + + public int statusCode() { + return this.httpStatusCode; + } + } + + static class ThrowableWithoutStatusCode extends Throwable {} + @BeforeEach public void setUpMocks() { errorCounterMock = mock(LongCounter.class); @@ -149,6 +180,16 @@ public void testOnEndMetricsGenerationWithLatency() { verify(latencyHistogramMock, times(1)).record(eq(5.5), eq(metricAttributes)); } + @Test + public void testOnEndMetricsGenerationWithAwsStatusCodes() { + validateMetricsGeneratedForAwsStatusCode(399L, ExpectedStatusMetric.NEITHER); + validateMetricsGeneratedForAwsStatusCode(400L, ExpectedStatusMetric.ERROR); + validateMetricsGeneratedForAwsStatusCode(499L, ExpectedStatusMetric.ERROR); + validateMetricsGeneratedForAwsStatusCode(500L, ExpectedStatusMetric.FAULT); + validateMetricsGeneratedForAwsStatusCode(599L, ExpectedStatusMetric.FAULT); + validateMetricsGeneratedForAwsStatusCode(600L, ExpectedStatusMetric.NEITHER); + } + @Test public void testOnEndMetricsGenerationWithStatusCodes() { // Invalid HTTP status codes @@ -192,6 +233,9 @@ private static ReadableSpan buildReadableSpanMock(Attributes spanAttributes) { // Configure spanData SpanData mockSpanData = mock(SpanData.class); + InstrumentationScopeInfo awsSdkScopeInfo = + InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build(); + when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo); when(mockSpanData.getAttributes()).thenReturn(spanAttributes); when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size()); when(readableSpanMock.toSpanData()).thenReturn(mockSpanData); @@ -199,6 +243,34 @@ private static ReadableSpan buildReadableSpanMock(Attributes spanAttributes) { return readableSpanMock; } + private static ReadableSpan buildReadableSpanWithThrowableMock(Throwable throwable) { + // config http status code as null + Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, null); + ReadableSpan readableSpanMock = mock(ReadableSpan.class); + SpanData mockSpanData = mock(SpanData.class); + InstrumentationScopeInfo awsSdkScopeInfo = + InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build(); + ExceptionEventData mockEventData = mock(ExceptionEventData.class); + List events = new ArrayList<>(Arrays.asList(mockEventData)); + + // Configure latency + when(readableSpanMock.getLatencyNanos()).thenReturn(TEST_LATENCY_NANOS); + + // Configure attributes + when(readableSpanMock.getAttribute(any())) + .thenAnswer(invocation -> spanAttributes.get(invocation.getArgument(0))); + + // Configure spanData + when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo); + when(mockSpanData.getAttributes()).thenReturn(spanAttributes); + when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size()); + when(mockSpanData.getEvents()).thenReturn(events); + when(mockEventData.getException()).thenReturn(throwable); + when(readableSpanMock.toSpanData()).thenReturn(mockSpanData); + + return readableSpanMock; + } + private void configureMocksForOnEnd(ReadableSpan readableSpanMock, Attributes metricAttributes) { // Configure generated attributes when(generatorMock.generateMetricAttributesFromSpan( @@ -214,6 +286,35 @@ private void validateMetricsGeneratedForHttpStatusCode( configureMocksForOnEnd(readableSpanMock, metricAttributes); awsSpanMetricsProcessor.onEnd(readableSpanMock); + validateMetrics(metricAttributes, expectedStatusMetric); + } + + private void validateMetricsGeneratedForAwsStatusCode( + Long awsStatusCode, ExpectedStatusMetric expectedStatusMetric) { + Throwable throwableWithMethodGetStatusCode = + new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue()); + validateMetricsGeneratedByThrowable(throwableWithMethodGetStatusCode, expectedStatusMetric); + + Throwable throwableWithMethodStatusCode = + new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue()); + validateMetricsGeneratedByThrowable(throwableWithMethodStatusCode, expectedStatusMetric); + + Throwable throwableWithoutStatusCode = new ThrowableWithoutStatusCode(); + validateMetricsGeneratedByThrowable(throwableWithoutStatusCode, ExpectedStatusMetric.NEITHER); + } + + private void validateMetricsGeneratedByThrowable( + Throwable throwable, ExpectedStatusMetric expectedStatusMetric) { + ReadableSpan readableSpanMock = buildReadableSpanWithThrowableMock(throwable); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, metricAttributes); + + awsSpanMetricsProcessor.onEnd(readableSpanMock); + validateMetrics(metricAttributes, expectedStatusMetric); + } + + private void validateMetrics( + Attributes metricAttributes, ExpectedStatusMetric expectedStatusMetric) { switch (expectedStatusMetric) { case ERROR: verify(errorCounterMock, times(1)).add(eq(1L), eq(metricAttributes));