Skip to content

Commit 314dadc

Browse files
authored
feat: Adding gfe_latencies metric to built-in metrics (#3490)
1 parent 4a1f99c commit 314dadc

15 files changed

+549
-68
lines changed

google-cloud-spanner/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@
371371
<artifactId>junit</artifactId>
372372
<scope>test</scope>
373373
</dependency>
374-
374+
375375
<!-- Executor tests - The 'provided' scope is overwritten to compile time scope for the profile 'executor-tests' -->
376376
<dependency>
377377
<groupId>com.google.api.grpc</groupId>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java

+18-4
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@
3434
public class BuiltInMetricsConstant {
3535

3636
public static final String METER_NAME = "spanner.googleapis.com/internal/client";
37-
3837
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
39-
38+
static final String SPANNER_METER_NAME = "spanner-java";
39+
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4040
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
4141
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
4242
static final String OPERATION_LATENCY_NAME = "operation_latency";
@@ -49,7 +49,8 @@ public class BuiltInMetricsConstant {
4949
OPERATION_LATENCIES_NAME,
5050
ATTEMPT_LATENCIES_NAME,
5151
OPERATION_COUNT_NAME,
52-
ATTEMPT_COUNT_NAME)
52+
ATTEMPT_COUNT_NAME,
53+
GFE_LATENCIES_NAME)
5354
.stream()
5455
.map(m -> METER_NAME + '/' + m)
5556
.collect(Collectors.toSet());
@@ -114,27 +115,39 @@ static Map<InstrumentSelector, View> getAllViews() {
114115
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
115116
defineView(
116117
views,
118+
BuiltInMetricsConstant.GAX_METER_NAME,
117119
BuiltInMetricsConstant.OPERATION_LATENCY_NAME,
118120
BuiltInMetricsConstant.OPERATION_LATENCIES_NAME,
119121
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
120122
InstrumentType.HISTOGRAM,
121123
"ms");
122124
defineView(
123125
views,
126+
BuiltInMetricsConstant.GAX_METER_NAME,
124127
BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME,
125128
BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME,
126129
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
127130
InstrumentType.HISTOGRAM,
128131
"ms");
129132
defineView(
130133
views,
134+
BuiltInMetricsConstant.SPANNER_METER_NAME,
135+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
136+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
137+
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
138+
InstrumentType.HISTOGRAM,
139+
"ms");
140+
defineView(
141+
views,
142+
BuiltInMetricsConstant.GAX_METER_NAME,
131143
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
132144
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
133145
Aggregation.sum(),
134146
InstrumentType.COUNTER,
135147
"1");
136148
defineView(
137149
views,
150+
BuiltInMetricsConstant.GAX_METER_NAME,
138151
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
139152
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
140153
Aggregation.sum(),
@@ -145,6 +158,7 @@ static Map<InstrumentSelector, View> getAllViews() {
145158

146159
private static void defineView(
147160
ImmutableMap.Builder<InstrumentSelector, View> viewMap,
161+
String meterName,
148162
String metricName,
149163
String metricViewName,
150164
Aggregation aggregation,
@@ -153,7 +167,7 @@ private static void defineView(
153167
InstrumentSelector selector =
154168
InstrumentSelector.builder()
155169
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName)
156-
.setMeterName(BuiltInMetricsConstant.GAX_METER_NAME)
170+
.setMeterName(meterName)
157171
.setType(type)
158172
.setUnit(unit)
159173
.build();

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -46,25 +46,24 @@
4646
import java.util.logging.Logger;
4747
import javax.annotation.Nullable;
4848

49-
final class BuiltInOpenTelemetryMetricsProvider {
49+
final class BuiltInMetricsProvider {
5050

51-
static BuiltInOpenTelemetryMetricsProvider INSTANCE = new BuiltInOpenTelemetryMetricsProvider();
51+
static BuiltInMetricsProvider INSTANCE = new BuiltInMetricsProvider();
5252

53-
private static final Logger logger =
54-
Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName());
53+
private static final Logger logger = Logger.getLogger(BuiltInMetricsProvider.class.getName());
5554

5655
private static String taskId;
5756

5857
private OpenTelemetry openTelemetry;
5958

60-
private BuiltInOpenTelemetryMetricsProvider() {}
59+
private BuiltInMetricsProvider() {}
6160

6261
OpenTelemetry getOrCreateOpenTelemetry(
6362
String projectId, @Nullable Credentials credentials, @Nullable String monitoringHost) {
6463
try {
6564
if (this.openTelemetry == null) {
6665
SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder();
67-
BuiltInOpenTelemetryMetricsView.registerBuiltinMetrics(
66+
BuiltInMetricsView.registerBuiltinMetrics(
6867
SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
6968
sdkMeterProviderBuilder);
7069
SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.core.GaxProperties;
20+
import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder;
21+
import com.google.common.base.Preconditions;
22+
import io.opentelemetry.api.OpenTelemetry;
23+
import io.opentelemetry.api.common.Attributes;
24+
import io.opentelemetry.api.common.AttributesBuilder;
25+
import io.opentelemetry.api.metrics.DoubleHistogram;
26+
import io.opentelemetry.api.metrics.Meter;
27+
import java.util.Map;
28+
29+
/**
30+
* Implementation for recording built in metrics.
31+
*
32+
* <p>This class extends the {@link OpenTelemetryMetricsRecorder} which implements the *
33+
* measurements related to the lifecyle of an RPC.
34+
*/
35+
class BuiltInMetricsRecorder extends OpenTelemetryMetricsRecorder {
36+
37+
private final DoubleHistogram gfeLatencyRecorder;
38+
39+
/**
40+
* Creates the following instruments for the following metrics:
41+
*
42+
* <ul>
43+
* <li>GFE Latency: Histogram
44+
* </ul>
45+
*
46+
* @param openTelemetry OpenTelemetry instance
47+
* @param serviceName Service Name
48+
*/
49+
BuiltInMetricsRecorder(OpenTelemetry openTelemetry, String serviceName) {
50+
super(openTelemetry, serviceName);
51+
Meter meter =
52+
openTelemetry
53+
.meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME)
54+
.setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass()))
55+
.build();
56+
this.gfeLatencyRecorder =
57+
meter
58+
.histogramBuilder(serviceName + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME)
59+
.setDescription(
60+
"Latency between Google's network receiving an RPC and reading back the first byte of the response")
61+
.setUnit("ms")
62+
.build();
63+
}
64+
65+
/**
66+
* Record the latency between Google's network receiving an RPC and reading back the first byte of
67+
* the response. Data is stored in a Histogram.
68+
*
69+
* @param gfeLatency Attempt Latency in ms
70+
* @param attributes Map of the attributes to store
71+
*/
72+
void recordGFELatency(double gfeLatency, Map<String, String> attributes) {
73+
gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(attributes));
74+
}
75+
76+
Attributes toOtelAttributes(Map<String, String> attributes) {
77+
Preconditions.checkNotNull(attributes, "Attributes map cannot be null");
78+
AttributesBuilder attributesBuilder = Attributes.builder();
79+
attributes.forEach(attributesBuilder::put);
80+
return attributesBuilder.build();
81+
}
82+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.rpc.ApiException;
20+
import com.google.api.gax.rpc.StatusCode;
21+
import com.google.api.gax.tracing.ApiTracer;
22+
import com.google.api.gax.tracing.MethodName;
23+
import com.google.api.gax.tracing.MetricsTracer;
24+
import java.util.HashMap;
25+
import java.util.Map;
26+
import java.util.concurrent.CancellationException;
27+
import javax.annotation.Nullable;
28+
29+
/**
30+
* Implements built-in metrics tracer.
31+
*
32+
* <p>This class extends the {@link MetricsTracer} which computes generic metrics that can be
33+
* observed in the lifecycle of an RPC operation.
34+
*/
35+
class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
36+
37+
private final BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder;
38+
// These are RPC specific attributes and pertain to a specific API Trace
39+
private final Map<String, String> attributes = new HashMap<>();
40+
41+
private Long gfeLatency = null;
42+
43+
BuiltInMetricsTracer(
44+
MethodName methodName, BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder) {
45+
super(methodName, builtInOpenTelemetryMetricsRecorder);
46+
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
47+
this.attributes.put(METHOD_ATTRIBUTE, methodName.toString());
48+
}
49+
50+
/**
51+
* Adds an annotation that the attempt succeeded. Successful attempt add "OK" value to the status
52+
* attribute key.
53+
*/
54+
@Override
55+
public void attemptSucceeded() {
56+
super.attemptSucceeded();
57+
if (gfeLatency != null) {
58+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
59+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
60+
}
61+
}
62+
63+
/**
64+
* Add an annotation that the attempt was cancelled by the user. Cancelled attempt add "CANCELLED"
65+
* to the status attribute key.
66+
*/
67+
@Override
68+
public void attemptCancelled() {
69+
super.attemptCancelled();
70+
if (gfeLatency != null) {
71+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
72+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
73+
}
74+
}
75+
76+
/**
77+
* Adds an annotation that the attempt failed, but another attempt will be made after the delay.
78+
*
79+
* @param error the error that caused the attempt to fail.
80+
* @param delay the amount of time to wait before the next attempt will start.
81+
* <p>Failed attempt extracts the error from the throwable and adds it to the status attribute
82+
* key.
83+
*/
84+
@Override
85+
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
86+
super.attemptFailedDuration(error, delay);
87+
if (gfeLatency != null) {
88+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
89+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
90+
}
91+
}
92+
93+
/**
94+
* Adds an annotation that the attempt failed and that no further attempts will be made because
95+
* retry limits have been reached. This extracts the error from the throwable and adds it to the
96+
* status attribute key.
97+
*
98+
* @param error the last error received before retries were exhausted.
99+
*/
100+
@Override
101+
public void attemptFailedRetriesExhausted(Throwable error) {
102+
super.attemptFailedRetriesExhausted(error);
103+
if (gfeLatency != null) {
104+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
105+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
106+
}
107+
}
108+
109+
/**
110+
* Adds an annotation that the attempt failed and that no further attempts will be made because
111+
* the last error was not retryable. This extracts the error from the throwable and adds it to the
112+
* status attribute key.
113+
*
114+
* @param error the error that caused the final attempt to fail.
115+
*/
116+
@Override
117+
public void attemptPermanentFailure(Throwable error) {
118+
super.attemptPermanentFailure(error);
119+
if (gfeLatency != null) {
120+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
121+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
122+
}
123+
}
124+
125+
void recordGFELatency(Long gfeLatency) {
126+
this.gfeLatency = gfeLatency;
127+
}
128+
129+
@Override
130+
public void addAttributes(Map<String, String> attributes) {
131+
super.addAttributes(attributes);
132+
this.attributes.putAll(attributes);
133+
};
134+
135+
@Override
136+
public void addAttributes(String key, String value) {
137+
super.addAttributes(key, value);
138+
this.attributes.put(key, value);
139+
}
140+
141+
private static String extractStatus(@Nullable Throwable error) {
142+
final String statusString;
143+
144+
if (error == null) {
145+
return StatusCode.Code.OK.toString();
146+
} else if (error instanceof CancellationException) {
147+
statusString = StatusCode.Code.CANCELLED.toString();
148+
} else if (error instanceof ApiException) {
149+
statusString = ((ApiException) error).getStatusCode().getCode().toString();
150+
} else {
151+
statusString = StatusCode.Code.UNKNOWN.toString();
152+
}
153+
154+
return statusString;
155+
}
156+
}

0 commit comments

Comments
 (0)