Skip to content

Commit

Permalink
generate error/fault metrics by aws sdk status code (#924)
Browse files Browse the repository at this point in the history
  • Loading branch information
@scaugrated
scaugrated committed Jul 3, 2023
1 parent 17d03c8 commit d9aaf2d
Show file tree
Hide file tree
Showing 2 changed files with 160 additions and 5 deletions.
62 changes: 58 additions & 4 deletions aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
import io.opentelemetry.sdk.trace.ReadWriteSpan;
import io.opentelemetry.sdk.trace.ReadableSpan;
import io.opentelemetry.sdk.trace.SpanProcessor;
import io.opentelemetry.sdk.trace.data.EventData;
import io.opentelemetry.sdk.trace.data.SpanData;
import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData;
import java.lang.reflect.Method;
import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;

/**
Expand Down Expand Up @@ -92,7 +96,7 @@ public void onEnd(ReadableSpan span) {

// Only record metrics if non-empty attributes are returned.
if (!attributes.isEmpty()) {
recordErrorOrFault(span, attributes);
recordErrorOrFault(spanData, attributes);
recordLatency(span, attributes);
}
}
Expand All @@ -102,10 +106,14 @@ public boolean isEndRequired() {
return true;
}

private void recordErrorOrFault(ReadableSpan span, Attributes attributes) {
Long httpStatusCode = span.getAttribute(HTTP_STATUS_CODE);
private void recordErrorOrFault(SpanData spanData, Attributes attributes) {
Long httpStatusCode = spanData.getAttributes().get(HTTP_STATUS_CODE);
if (httpStatusCode == null) {
return;
httpStatusCode = getAwsStatusCode(spanData);

if (httpStatusCode == null || httpStatusCode < 100L || httpStatusCode > 599L) {
return;
}
}

if (httpStatusCode >= ERROR_CODE_LOWER_BOUND && httpStatusCode <= ERROR_CODE_UPPER_BOUND) {
Expand All @@ -116,6 +124,52 @@ private void recordErrorOrFault(ReadableSpan span, Attributes attributes) {
}
}

/**
* Attempt to pull status code from spans produced by AWS SDK instrumentation (both v1 and v2).
* AWS SDK instrumentation does not populate http.status_code when non-200 status codes are
* returned, as the AWS SDK throws exceptions rather than returning responses with status codes.
* To work around this, we are attempting to get the exception out of the events, then calling
* getStatusCode (for AWS SDK V1) and statusCode (for AWS SDK V2) to get the status code fromt the
* exception. We rely on reflection here because we cannot cast the throwable to
* AmazonServiceExceptions (V1) or AwsServiceExceptions (V2) because the throwable comes from a
* separate class loader and attempts to cast will fail with ClassCastException.
*
* <p>TODO: Short term workaround. This can be completely removed once
* https://github.com/open-telemetry/opentelemetry-java-contrib/issues/919 is resolved.
*/
@Nullable
private static Long getAwsStatusCode(SpanData spanData) {
String scopeName = spanData.getInstrumentationScopeInfo().getName();
if (!scopeName.contains("aws-sdk")) {
return null;
}

for (EventData event : spanData.getEvents()) {
if (event instanceof ExceptionEventData) {
ExceptionEventData exceptionEvent = (ExceptionEventData) event;
Throwable throwable = exceptionEvent.getException();

try {
Method method = throwable.getClass().getMethod("getStatusCode", new Class<?>[] {});
Object code = method.invoke(throwable, new Object[] {});
return Long.valueOf((Integer) code);
} catch (Exception e) {
// Take no action
}

try {
Method method = throwable.getClass().getMethod("statusCode", new Class<?>[] {});
Object code = method.invoke(throwable, new Object[] {});
return Long.valueOf((Integer) code);
} catch (Exception e) {
// Take no action
}
}
}

return null;
}

private void recordLatency(ReadableSpan span, Attributes attributes) {
long nanos = span.getLatencyNanos();
double millis = nanos / NANOS_TO_MILLIS;
Expand Down
103 changes: 102 additions & 1 deletion aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,21 @@
import io.opentelemetry.api.metrics.LongCounter;
import io.opentelemetry.context.Context;
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.ReadWriteSpan;
import io.opentelemetry.sdk.trace.ReadableSpan;
import io.opentelemetry.sdk.trace.data.EventData;
import io.opentelemetry.sdk.trace.data.SpanData;
import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

/** Unit tests for {@link AwsSpanMetricsProcessor}. */
class AwsSpanMetricsProcessorTest {

// Test constants
private static final boolean CONTAINS_ATTRIBUTES = true;
private static final boolean CONTAINS_NO_ATTRIBUTES = false;
Expand All @@ -56,6 +61,32 @@ private enum ExpectedStatusMetric {

private AwsSpanMetricsProcessor awsSpanMetricsProcessor;

static class ThrowableWithMethodGetStatusCode extends Throwable {
private final int httpStatusCode;

ThrowableWithMethodGetStatusCode(int httpStatusCode) {
this.httpStatusCode = httpStatusCode;
}

public int getStatusCode() {
return this.httpStatusCode;
}
}

static class ThrowableWithMethodStatusCode extends Throwable {
private final int httpStatusCode;

ThrowableWithMethodStatusCode(int httpStatusCode) {
this.httpStatusCode = httpStatusCode;
}

public int statusCode() {
return this.httpStatusCode;
}
}

static class ThrowableWithoutStatusCode extends Throwable {}

@BeforeEach
public void setUpMocks() {
errorCounterMock = mock(LongCounter.class);
Expand Down Expand Up @@ -149,6 +180,16 @@ public void testOnEndMetricsGenerationWithLatency() {
verify(latencyHistogramMock, times(1)).record(eq(5.5), eq(metricAttributes));
}

@Test
public void testOnEndMetricsGenerationWithAwsStatusCodes() {
validateMetricsGeneratedForAwsStatusCode(399L, ExpectedStatusMetric.NEITHER);
validateMetricsGeneratedForAwsStatusCode(400L, ExpectedStatusMetric.ERROR);
validateMetricsGeneratedForAwsStatusCode(499L, ExpectedStatusMetric.ERROR);
validateMetricsGeneratedForAwsStatusCode(500L, ExpectedStatusMetric.FAULT);
validateMetricsGeneratedForAwsStatusCode(599L, ExpectedStatusMetric.FAULT);
validateMetricsGeneratedForAwsStatusCode(600L, ExpectedStatusMetric.NEITHER);
}

@Test
public void testOnEndMetricsGenerationWithStatusCodes() {
// Invalid HTTP status codes
Expand Down Expand Up @@ -192,13 +233,44 @@ private static ReadableSpan buildReadableSpanMock(Attributes spanAttributes) {

// Configure spanData
SpanData mockSpanData = mock(SpanData.class);
InstrumentationScopeInfo awsSdkScopeInfo =
InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build();
when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo);
when(mockSpanData.getAttributes()).thenReturn(spanAttributes);
when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size());
when(readableSpanMock.toSpanData()).thenReturn(mockSpanData);

return readableSpanMock;
}

private static ReadableSpan buildReadableSpanWithThrowableMock(Throwable throwable) {
// config http status code as null
Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, null);
ReadableSpan readableSpanMock = mock(ReadableSpan.class);
SpanData mockSpanData = mock(SpanData.class);
InstrumentationScopeInfo awsSdkScopeInfo =
InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build();
ExceptionEventData mockEventData = mock(ExceptionEventData.class);
List<EventData> events = new ArrayList<>(Arrays.asList(mockEventData));

// Configure latency
when(readableSpanMock.getLatencyNanos()).thenReturn(TEST_LATENCY_NANOS);

// Configure attributes
when(readableSpanMock.getAttribute(any()))
.thenAnswer(invocation -> spanAttributes.get(invocation.getArgument(0)));

// Configure spanData
when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo);
when(mockSpanData.getAttributes()).thenReturn(spanAttributes);
when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size());
when(mockSpanData.getEvents()).thenReturn(events);
when(mockEventData.getException()).thenReturn(throwable);
when(readableSpanMock.toSpanData()).thenReturn(mockSpanData);

return readableSpanMock;
}

private void configureMocksForOnEnd(ReadableSpan readableSpanMock, Attributes metricAttributes) {
// Configure generated attributes
when(generatorMock.generateMetricAttributesFromSpan(
Expand All @@ -214,6 +286,35 @@ private void validateMetricsGeneratedForHttpStatusCode(
configureMocksForOnEnd(readableSpanMock, metricAttributes);

awsSpanMetricsProcessor.onEnd(readableSpanMock);
validateMetrics(metricAttributes, expectedStatusMetric);
}

private void validateMetricsGeneratedForAwsStatusCode(
Long awsStatusCode, ExpectedStatusMetric expectedStatusMetric) {
Throwable throwableWithMethodGetStatusCode =
new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue());
validateMetricsGeneratedByThrowable(throwableWithMethodGetStatusCode, expectedStatusMetric);

Throwable throwableWithMethodStatusCode =
new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue());
validateMetricsGeneratedByThrowable(throwableWithMethodStatusCode, expectedStatusMetric);

Throwable throwableWithoutStatusCode = new ThrowableWithoutStatusCode();
validateMetricsGeneratedByThrowable(throwableWithoutStatusCode, ExpectedStatusMetric.NEITHER);
}

private void validateMetricsGeneratedByThrowable(
Throwable throwable, ExpectedStatusMetric expectedStatusMetric) {
ReadableSpan readableSpanMock = buildReadableSpanWithThrowableMock(throwable);
Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES);
configureMocksForOnEnd(readableSpanMock, metricAttributes);

awsSpanMetricsProcessor.onEnd(readableSpanMock);
validateMetrics(metricAttributes, expectedStatusMetric);
}

private void validateMetrics(
Attributes metricAttributes, ExpectedStatusMetric expectedStatusMetric) {
switch (expectedStatusMetric) {
case ERROR:
verify(errorCounterMock, times(1)).add(eq(1L), eq(metricAttributes));
Expand Down

0 comments on commit d9aaf2d

Please sign in to comment.