Skip to content

Commit 2e02cfc

Browse files
cataphractcataphract
authored
impl rfc "APM endpoint resource renaming in the tracers" (#1219)
impl rfc "APM endpoint resource renaming in the tracers" Address review comments Try fix to test failure
1 parent 0cc3f7f commit 2e02cfc

File tree

25 files changed

+992
-22
lines changed

25 files changed

+992
-22
lines changed

‎.github/workflows/diff-proto-files.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ fi
3333
GO_AGENT_PROTO=$(curl -s "https://raw.githubusercontent.com/DataDog/datadog-agent/$DATADOG_AGENT_TAG/pkg/proto/datadog/trace/$PROTO_FILE")
3434
FIX_IMPORT_PATH=$(echo "$GO_AGENT_PROTO" | sed -e 's/import "datadog\/trace\//import "/g')
3535
FIX_PACKAGE_NAME=$(echo "$FIX_IMPORT_PATH" | sed -e 's/datadog\.trace/pb/g')
36-
echo "$FIX_PACKAGE_NAME" | diff "$PROTO_FILE" -
36+
echo "$FIX_PACKAGE_NAME" | diff -u "$PROTO_FILE" -

‎.github/workflows/verify-proto-files.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
types: [ opened, synchronize, reopened ]
55

66
env:
7-
DATADOG_AGENT_TAG: "7.55.0-rc.3"
7+
DATADOG_AGENT_TAG: "7f6d07c93ba087f23f80a3f0c2da4b1f3dc664d7"
88
rust_version: "1.84.1"
99

1010
jobs:
@@ -35,6 +35,16 @@ jobs:
3535
working-directory: datadog-trace-protobuf/src/pb
3636
run: |
3737
../../../.github/workflows/diff-proto-files.sh --file span.proto --tag ${{ env.DATADOG_AGENT_TAG }}
38+
- name: diff idx/tracer_payload.proto
39+
if: success() || failure()
40+
working-directory: datadog-trace-protobuf/src/pb
41+
run: |
42+
../../../.github/workflows/diff-proto-files.sh --file idx/tracer_payload.proto --tag ${{ env.DATADOG_AGENT_TAG }}
43+
- name: diff idx/span.proto
44+
if: success() || failure()
45+
working-directory: datadog-trace-protobuf/src/pb
46+
run: |
47+
../../../.github/workflows/diff-proto-files.sh --file idx/span.proto --tag ${{ env.DATADOG_AGENT_TAG }}
3848
- name: Cache
3949
uses: ./.github/actions/cache
4050
with:

‎data-pipeline/src/span_concentrator/aggregation.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@ pub(super) struct AggregationKey<'a> {
2828
is_synthetics_request: bool,
2929
peer_tags: Vec<(Cow<'a, str>, Cow<'a, str>)>,
3030
is_trace_root: bool,
31+
http_method: Cow<'a, str>,
32+
http_endpoint: Cow<'a, str>,
3133
}
3234

3335
/// Common representation of AggregationKey used to compare AggregationKey with different lifetimes
36+
/// field order must be the same as in AggregationKey, o/wise hashes will be different
3437
#[derive(Clone, Hash, PartialEq, Eq)]
3538
pub(super) struct BorrowedAggregationKey<'a> {
3639
resource_name: &'a str,
@@ -42,6 +45,8 @@ pub(super) struct BorrowedAggregationKey<'a> {
4245
is_synthetics_request: bool,
4346
peer_tags: Vec<(&'a str, &'a str)>,
4447
is_trace_root: bool,
48+
http_method: &'a str,
49+
http_endpoint: &'a str,
4550
}
4651

4752
/// Trait used to define a common type (`dyn BorrowableAggregationKey`) for all AggregationKey
@@ -71,6 +76,8 @@ impl BorrowableAggregationKey for AggregationKey<'_> {
7176
.map(|(tag, value)| (tag.borrow(), value.borrow()))
7277
.collect(),
7378
is_trace_root: self.is_trace_root,
79+
http_method: self.http_method.borrow(),
80+
http_endpoint: self.http_endpoint.borrow(),
7481
}
7582
}
7683
}
@@ -124,13 +131,29 @@ impl<'a> AggregationKey<'a> {
124131
} else {
125132
vec![]
126133
};
134+
135+
let http_method = span
136+
.meta
137+
.get("http.method")
138+
.map(|s| s.borrow())
139+
.unwrap_or_default();
140+
141+
let http_endpoint = span
142+
.meta
143+
.get("http.endpoint")
144+
.or_else(|| span.meta.get("http.route"))
145+
.map(|s| s.borrow())
146+
.unwrap_or_default();
147+
127148
Self {
128149
resource_name: span.resource.borrow().into(),
129150
service_name: span.service.borrow().into(),
130151
operation_name: span.name.borrow().into(),
131152
span_type: span.r#type.borrow().into(),
132153
span_kind: span_kind.into(),
133154
http_status_code: get_status_code(span),
155+
http_method: http_method.into(),
156+
http_endpoint: http_endpoint.into(),
134157
is_synthetics_request: span
135158
.meta
136159
.get(TAG_ORIGIN)
@@ -153,6 +176,8 @@ impl<'a> AggregationKey<'a> {
153176
span_type: Cow::Owned(self.span_type.into_owned()),
154177
span_kind: Cow::Owned(self.span_kind.into_owned()),
155178
http_status_code: self.http_status_code,
179+
http_method: Cow::Owned(self.http_method.into_owned()),
180+
http_endpoint: Cow::Owned(self.http_endpoint.into_owned()),
156181
is_synthetics_request: self.is_synthetics_request,
157182
is_trace_root: self.is_trace_root,
158183
peer_tags: self
@@ -183,6 +208,8 @@ impl From<pb::ClientGroupedStats> for AggregationKey<'static> {
183208
})
184209
.collect(),
185210
is_trace_root: value.is_trace_root == 1,
211+
http_method: value.http_method.into(),
212+
http_endpoint: value.http_endpoint.into(),
186213
}
187214
}
188215
}
@@ -335,6 +362,9 @@ fn encode_grouped_stats(key: AggregationKey, group: GroupedStats) -> pb::ClientG
335362
} else {
336363
pb::Trilean::False.into()
337364
},
365+
http_method: key.http_method.into_owned(),
366+
http_endpoint: key.http_endpoint.into_owned(),
367+
grpc_status_code: String::new(), // currently not used
338368
}
339369
}
340370

@@ -541,6 +571,57 @@ mod tests {
541571
..Default::default()
542572
},
543573
),
574+
// Span with http.method and http.route
575+
(
576+
SpanBytes {
577+
service: "service".into(),
578+
name: "op".into(),
579+
resource: "GET /api/v1/users".into(),
580+
span_id: 1,
581+
parent_id: 0,
582+
meta: HashMap::from([
583+
("http.method".into(), "GET".into()),
584+
("http.route".into(), "/api/v1/users".into()),
585+
]),
586+
..Default::default()
587+
},
588+
AggregationKey {
589+
service_name: "service".into(),
590+
operation_name: "op".into(),
591+
resource_name: "GET /api/v1/users".into(),
592+
http_method: "GET".into(),
593+
http_endpoint: "/api/v1/users".into(),
594+
is_synthetics_request: false,
595+
is_trace_root: true,
596+
..Default::default()
597+
},
598+
),
599+
// Span with http.method and http.endpoint (http.endpoint takes precedence)
600+
(
601+
SpanBytes {
602+
service: "service".into(),
603+
name: "op".into(),
604+
resource: "POST /users/create".into(),
605+
span_id: 1,
606+
parent_id: 0,
607+
meta: HashMap::from([
608+
("http.method".into(), "POST".into()),
609+
("http.route".into(), "/users/create".into()),
610+
("http.endpoint".into(), "/users/create2".into()),
611+
]),
612+
..Default::default()
613+
},
614+
AggregationKey {
615+
service_name: "service".into(),
616+
operation_name: "op".into(),
617+
resource_name: "POST /users/create".into(),
618+
http_method: "POST".into(),
619+
http_endpoint: "/users/create2".into(),
620+
is_synthetics_request: false,
621+
is_trace_root: true,
622+
..Default::default()
623+
},
624+
),
544625
];
545626

546627
let test_peer_tags = vec![

‎data-pipeline/src/stats_exporter.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,8 @@ fn encode_stats_payload(
171171
tags: Vec::new(),
172172
agent_aggregation: String::new(),
173173
image_tag: String::new(),
174+
process_tags: String::new(),
175+
process_tags_hash: 0,
174176
}
175177
}
176178

‎datadog-ipc/tarpc/src/serde_transport.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -461,11 +461,11 @@ pub mod unix {
461461

462462
#[test]
463463
fn temp_path_buf_non_random() {
464-
let sock = TempPathBuf::new("test");
464+
let sock = TempPathBuf::new("test_non_random");
465465
let mut good = std::env::temp_dir();
466-
good.push("test");
466+
good.push("test_non_random");
467467
assert_eq!(sock.as_ref(), good);
468-
assert_eq!(sock.as_ref().file_name().unwrap(), "test");
468+
assert_eq!(sock.as_ref().file_name().unwrap(), "test_non_random");
469469
}
470470

471471
#[test]
@@ -510,9 +510,9 @@ pub mod unix {
510510
#[test]
511511
fn temp_path_buf_preexisting_file() {
512512
let mut pre_existing = std::env::temp_dir();
513-
pre_existing.push("test");
513+
pre_existing.push("test_preexisting");
514514
let _file = std::fs::File::create(&pre_existing).unwrap();
515-
let sock = TempPathBuf::new("test");
515+
let sock = TempPathBuf::new("test_preexisting");
516516
let sock_path = std::path::PathBuf::from(sock.as_ref());
517517
assert!(sock_path.exists());
518518
std::mem::drop(sock);

‎datadog-trace-normalization/benches/normalization_utils.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ fn normalize_span_bench(c: &mut Criterion) {
105105
r#type: "http".to_string(),
106106
meta_struct: HashMap::new(),
107107
span_links: vec![],
108+
span_events: vec![],
108109
},
109110
pb::Span {
110111
duration: 12000000,
@@ -124,6 +125,7 @@ fn normalize_span_bench(c: &mut Criterion) {
124125
r#type: "http".to_string(),
125126
meta_struct: HashMap::new(),
126127
span_links: vec![],
128+
span_events: vec![],
127129
},
128130
];
129131

‎datadog-trace-normalization/src/normalizer.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ mod tests {
138138
r#type: "http".to_string(),
139139
meta_struct: HashMap::new(),
140140
span_links: vec![],
141+
span_events: vec![],
141142
}
142143
}
143144

‎datadog-trace-obfuscation/benches/benchmarks/replace_trace_tags_bench.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ fn criterion_benchmark(c: &mut Criterion) {
4646
r#type: "http".to_string(),
4747
meta_struct: HashMap::new(),
4848
span_links: vec![],
49+
span_events: vec![],
4950
};
5051

5152
let trace = [span_1];

‎datadog-trace-obfuscation/src/replacer.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ mod tests {
188188
r#type: "http".to_string(),
189189
meta_struct: HashMap::new(),
190190
span_links: vec![],
191+
span_events: vec![],
191192
};
192193
for (key, val) in tags {
193194
match key {

‎datadog-trace-protobuf/build.rs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,33 @@ fn generate_protobuf() {
115115
".pb.Span.error",
116116
"#[serde(default)] #[serde(deserialize_with = \"crate::deserializers::deserialize_null_into_default\")] #[serde(skip_serializing_if = \"crate::deserializers::is_default\")]",
117117
);
118+
config.field_attribute(
119+
".pb.Span.spanEvents",
120+
"#[serde(default)] #[serde(deserialize_with = \"crate::deserializers::deserialize_null_into_default\")] #[serde(skip_serializing_if = \"::prost::alloc::vec::Vec::is_empty\")]",
121+
);
122+
123+
config.type_attribute("SpanEvent", "#[derive(Deserialize, Serialize)]");
124+
config.field_attribute(".pb.SpanEvent.time_unix_nano", "#[serde(default)]");
125+
config.field_attribute(".pb.SpanEvent.name", "#[serde(default)]");
126+
config.field_attribute(".pb.SpanEvent.attributes", "#[serde(default)]");
127+
128+
config.type_attribute("AttributeAnyValue", "#[derive(Deserialize, Serialize)]");
129+
config.field_attribute(".pb.AttributeAnyValue.type", "#[serde(default)]");
130+
config.field_attribute(".pb.AttributeAnyValue.string_value", "#[serde(default)]");
131+
config.field_attribute(".pb.AttributeAnyValue.bool_value", "#[serde(default)]");
132+
config.field_attribute(".pb.AttributeAnyValue.int_value", "#[serde(default)]");
133+
config.field_attribute(".pb.AttributeAnyValue.double_value", "#[serde(default)]");
134+
config.field_attribute(".pb.AttributeAnyValue.array_value", "#[serde(default)]");
135+
136+
config.type_attribute("AttributeArray", "#[derive(Deserialize, Serialize)]");
137+
config.field_attribute(".pb.AttributeArray.values", "#[serde(default)]");
138+
139+
config.type_attribute("AttributeArrayValue", "#[derive(Deserialize, Serialize)]");
140+
config.field_attribute(".pb.AttributeArrayValue.type", "#[serde(default)]");
141+
config.field_attribute(".pb.AttributeArrayValue.string_value", "#[serde(default)]");
142+
config.field_attribute(".pb.AttributeArrayValue.bool_value", "#[serde(default)]");
143+
config.field_attribute(".pb.AttributeArrayValue.int_value", "#[serde(default)]");
144+
config.field_attribute(".pb.AttributeArrayValue.double_value", "#[serde(default)]");
118145

119146
config.type_attribute("StatsPayload", "#[derive(Deserialize, Serialize)]");
120147
config.type_attribute("StatsPayload", "#[serde(rename_all = \"PascalCase\")]");
@@ -140,6 +167,9 @@ fn generate_protobuf() {
140167
config.field_attribute("ClientGroupedStats.span_kind", "#[serde(default)]");
141168
config.field_attribute("ClientGroupedStats.peer_tags", "#[serde(default)]");
142169
config.field_attribute("ClientGroupedStats.is_trace_root", "#[serde(default)]");
170+
config.field_attribute("ClientGroupedStats.GRPC_status_code", "#[serde(default)]");
171+
config.field_attribute("ClientGroupedStats.HTTP_method", "#[serde(default)]");
172+
config.field_attribute("ClientGroupedStats.HTTP_endpoint", "#[serde(default)]");
143173

144174
config.field_attribute(
145175
"ClientGroupedStats.okSummary",
@@ -167,6 +197,16 @@ fn generate_protobuf() {
167197
"#[serde(rename = \"DBType\")]",
168198
);
169199

200+
// idx module type attributes
201+
config.type_attribute("pb.idx.AnyValue", "#[derive(Deserialize, Serialize)]");
202+
config.type_attribute(
203+
"pb.idx.AnyValue.value",
204+
"#[derive(serde::Deserialize, serde::Serialize)]",
205+
);
206+
config.type_attribute("pb.idx.KeyValue", "#[derive(Deserialize, Serialize)]");
207+
config.type_attribute("pb.idx.ArrayValue", "#[derive(Deserialize, Serialize)]");
208+
config.type_attribute("pb.idx.KeyValueList", "#[derive(Deserialize, Serialize)]");
209+
170210
config.type_attribute(
171211
"ClientGetConfigsResponse",
172212
"#[derive(Deserialize, Serialize)]",
@@ -205,6 +245,8 @@ fn generate_protobuf() {
205245
"#[serde(default)]",
206246
);
207247

248+
config.include_file("_includes.rs");
249+
208250
config
209251
.compile_protos(
210252
&[
@@ -213,6 +255,8 @@ fn generate_protobuf() {
213255
"src/pb/span.proto",
214256
"src/pb/stats.proto",
215257
"src/pb/remoteconfig.proto",
258+
"src/pb/idx/tracer_payload.proto",
259+
"src/pb/idx/span.proto",
216260
],
217261
&["src/pb/"],
218262
)
@@ -234,8 +278,10 @@ fn generate_protobuf() {
234278
]
235279
.concat();
236280

281+
prepend_to_file(license, &output_path.join("_includes.rs"));
237282
prepend_to_file(serde_uses, &output_path.join("pb.rs"));
238283
prepend_to_file(serde_uses, &output_path.join("remoteconfig.rs"));
284+
prepend_to_file(serde_uses, &output_path.join("pb.idx.rs"));
239285
}
240286

241287
#[cfg(feature = "generate-protobuf")]

0 commit comments

Comments
 (0)