datadog: metrics improvements and trace correlations (#2098)

odigos-io · Jan 2, 2025 · b3e7faa · b3e7faa
1 parent 7ad98e8
commit b3e7faa
Show file tree

Hide file tree

Showing 11 changed files with 85 additions and 8 deletions.
diff --git a/autoscaler/controllers/datacollection/configmap.go b/autoscaler/controllers/datacollection/configmap.go
@@ -344,8 +344,45 @@ func calculateConfigMapData(nodeCG *odigosv1.CollectorsGroup, sources *odigosv1.
 			"collection_interval":  "10s",
 		}
 
+		cfg.Receivers["hostmetrics"] = config.GenericMap{
+			"collection_interval": "10s",
+			"root_path":           "/hostfs",
+			"scrapers": config.GenericMap{
+				"paging": config.GenericMap{
+					"metrics": config.GenericMap{
+						"system.paging.utilization": config.GenericMap{
+							"enabled": true,
+						},
+					},
+				},
+				"cpu": config.GenericMap{
+					"metrics": config.GenericMap{
+						"system.cpu.utilization": config.GenericMap{
+							"enabled": true,
+						},
+					},
+				},
+				"disk": struct{}{},
+				"filesystem": config.GenericMap{
+					"metrics": config.GenericMap{
+						"system.filesystem.utilization": config.GenericMap{
+							"enabled": true,
+						},
+					},
+					"exclude_mount_points": config.GenericMap{
+						"match_type":   "regexp",
+						"mount_points": []string{"/var/lib/kubelet/*"},
+					},
+				},
+				"load":      struct{}{},
+				"memory":    struct{}{},
+				"network":   struct{}{},
+				"processes": struct{}{},
+			},
+		}
+
 		cfg.Service.Pipelines["metrics"] = config.Pipeline{
-			Receivers:  []string{"otlp", "kubeletstats"},
+			Receivers:  []string{"otlp", "kubeletstats", "hostmetrics"},
 			Processors: append(commonProcessors, metricsProcessors...),
 			Exporters:  []string{"otlp/gateway"},
 		}

diff --git a/autoscaler/controllers/datacollection/daemonset.go b/autoscaler/controllers/datacollection/daemonset.go
@@ -261,6 +261,14 @@ func getDesiredDaemonSet(datacollection *odigosv1.CollectorsGroup,
 								},
 							},
 						},
+						{
+							Name: "hostfs",
+							VolumeSource: corev1.VolumeSource{
+								HostPath: &corev1.HostPathVolumeSource{
+									Path: "/",
+								},
+							},
+						},
 					},
 					Containers: []corev1.Container{
 						{
@@ -287,6 +295,11 @@ func getDesiredDaemonSet(datacollection *odigosv1.CollectorsGroup,
 									MountPath: "/var/lib/kubelet/pod-resources",
 									ReadOnly:  true,
 								},
+								{
+									Name:      "hostfs",
+									MountPath: "/hostfs",
+									ReadOnly:  true,
+								},
 							},
 							Env: []corev1.EnvVar{
 								{

diff --git a/collector/builder-config.yaml b/collector/builder-config.yaml
@@ -91,6 +91,7 @@ receivers:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.106.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.106.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.106.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.106.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.106.0
 
 # https://github.com/open-telemetry/opentelemetry-collector/issues/8127

diff --git a/collector/odigosotelcol/components.go b/collector/odigosotelcol/components.go
diff --git a/collector/odigosotelcol/go.mod b/collector/odigosotelcol/go.mod
@@ -79,6 +79,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.106.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.106.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.106.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.106.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.106.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.106.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.106.0
@@ -451,6 +452,7 @@ require (
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/prometheus-community/windows_exporter v0.25.1 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect

diff --git a/common/config/config.go b/common/config/config.go
@@ -31,19 +31,19 @@ type Config struct {
 	Exporters  GenericMap `json:"exporters"`
 	Processors GenericMap `json:"processors"`
 	Extensions GenericMap `json:"extensions"`
-	Connectors GenericMap `json:"connectors"`
+	Connectors GenericMap `json:"connectors,omitempty"`
 	Service    Service    `json:"service"`
 }
 
 type Telemetry struct {
-	Metrics GenericMap `json:"metrics"`
+	Metrics  GenericMap         `json:"metrics"`
 	Resource map[string]*string `json:"resource"`
 }
 
 type Service struct {
 	Extensions []string            `json:"extensions"`
 	Pipelines  map[string]Pipeline `json:"pipelines"`
-	Telemetry Telemetry            `json:"telemetry,omitempty"`
+	Telemetry  Telemetry           `json:"telemetry,omitempty"`
 }
 
 type Pipeline struct {

diff --git a/common/config/datadog.go b/common/config/datadog.go
@@ -35,16 +35,33 @@ func (d *Datadog) ModifyConfig(dest ExporterConfigurer, currentConfig *Config) e
 		},
 	}
 
+	connectorEnabled := false
+	connectorName := "datadog/connector-" + dest.GetID()
+	if isTracingEnabled(dest) && isMetricsEnabled(dest) {
+		currentConfig.Connectors[connectorName] = struct{}{}
+		connectorEnabled = true
+	}
+
 	if isTracingEnabled(dest) {
 		tracesPipelineName := "traces/datadog-" + dest.GetID()
+		exporters := []string{exporterName}
+		if connectorEnabled {
+			exporters = append(exporters, connectorName)
+		}
+
 		currentConfig.Service.Pipelines[tracesPipelineName] = Pipeline{
-			Exporters: []string{exporterName},
+			Exporters: exporters,
 		}
 	}
 
 	if isMetricsEnabled(dest) {
 		metricsPipelineName := "metrics/datadog-" + dest.GetID()
+		var receivers []string
+		if connectorEnabled {
+			receivers = []string{connectorName}
+		}
 		currentConfig.Service.Pipelines[metricsPipelineName] = Pipeline{
+			Receivers: receivers,
 			Exporters: []string{exporterName},
 		}
 	}

diff --git a/common/config/testdata/debugexporter.yaml b/common/config/testdata/debugexporter.yaml
@@ -24,7 +24,6 @@ extensions:
   health_check:
     endpoint: 0.0.0.0:13133
   pprof: {}
-connectors: {}
 service:
   extensions:
   - health_check

diff --git a/common/config/testdata/minimal.yaml b/common/config/testdata/minimal.yaml
@@ -22,7 +22,6 @@ extensions:
   health_check:
     endpoint: 0.0.0.0:13133
   pprof: {}
-connectors: {}
 service:
   extensions:
   - health_check

diff --git a/common/config/testdata/withbaseminimal.yaml b/common/config/testdata/withbaseminimal.yaml
@@ -7,7 +7,6 @@ exporters: {}
 processors:
   batch: {}
 extensions: {}
-connectors: {}
 service:
   extensions: []
   pipelines: {}
diff --git a/tests/common/assert/pipeline-ready.yaml b/tests/common/assert/pipeline-ready.yaml
@@ -149,6 +149,9 @@ spec:
             - mountPath: /var/lib/kubelet/pod-resources
               name: kubeletpodresources
               readOnly: true
+            - mountPath: /hostfs
+              name: hostfs
+              readOnly: true
           env:
             - name: NODE_NAME
               valueFrom:
@@ -193,6 +196,10 @@ spec:
             path: /var/lib/kubelet/pod-resources
             type: ""
           name: kubeletpodresources
+        - hostPath:
+            path: /
+            type: ""
+          name: hostfs
 status:
   numberAvailable: 1
   numberReady: 1