From 5e908f796cc965eb07b1a13e23bf7415f0c32cb0 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Thu, 28 Mar 2024 11:27:39 +0000 Subject: [PATCH 1/4] add extra filter in prometheus scrape for http-metrics Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/agent/config.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 6e6c3c0..19db85f 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -85,11 +85,21 @@ data: name = "metrics" } + discovery.relabel "only_http_metrics" { + targets = discovery.relabel.rename_meta_labels.output + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + action = "keep" + regex = "http-metrics" + } + } + prometheus.scrape "pods" { clustering { enabled = true } - targets = discovery.relabel.rename_meta_labels.output + targets = discovery.relabel.only_http_metrics.output forward_to = [ {{ include "agent.prometheus_write_targets" . }} ] } {{- if .Values.kubeStateMetrics.enabled }} From 18a5face8189563916d769b65c6e04f5c863d93b Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Thu, 28 Mar 2024 14:14:25 +0000 Subject: [PATCH 2/4] cleanup comments Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/agent/config.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 19db85f..3290dc8 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -37,9 +37,9 @@ data: } } + {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }} // Logs - {{- if or .Values.local.logs.enabled .Values.cloud.logs.enabled }} remote.kubernetes.secret "logs_credentials" { namespace = "{{- .Values.metaMonitoringNamespace -}}" name = "logs" @@ -77,14 +77,18 @@ data: {{- end }} {{- end }} + {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} // Metrics +<<<<<<< HEAD {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} remote.kubernetes.secret "metrics_credentials" { namespace = "{{- .Values.metaMonitoringNamespace -}}" name = "metrics" } +======= +>>>>>>> bb84e9f (cleanup comments) discovery.relabel "only_http_metrics" { targets = discovery.relabel.rename_meta_labels.output @@ -227,14 +231,18 @@ data: } {{- end }} + {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }} // Traces +<<<<<<< HEAD {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }} remote.kubernetes.secret "traces_credentials" { namespace = "{{- .Values.metaMonitoringNamespace -}}" name = "traces" } +======= +>>>>>>> bb84e9f (cleanup comments) // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river otelcol.receiver.otlp "otlp_receiver" { // We don't technically need this, but it shows how to change listen address and incoming port. From b37fa4adf5ab17ecf1a4c7d2c96117572ba2eef1 Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 3 Apr 2024 12:46:34 +0100 Subject: [PATCH 3/4] Fix rebase Signed-off-by: Michel Hollands --- charts/meta-monitoring/templates/agent/config.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 3290dc8..8f1df2b 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -80,15 +80,11 @@ data: {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} // Metrics -<<<<<<< HEAD - {{- if or .Values.local.metrics.enabled .Values.cloud.metrics.enabled }} remote.kubernetes.secret "metrics_credentials" { namespace = "{{- .Values.metaMonitoringNamespace -}}" name = "metrics" } -======= ->>>>>>> bb84e9f (cleanup comments) discovery.relabel "only_http_metrics" { targets = discovery.relabel.rename_meta_labels.output @@ -234,15 +230,11 @@ data: {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }} // Traces -<<<<<<< HEAD - {{- if or .Values.local.traces.enabled .Values.cloud.traces.enabled }} remote.kubernetes.secret "traces_credentials" { namespace = "{{- .Values.metaMonitoringNamespace -}}" name = "traces" } -======= ->>>>>>> bb84e9f (cleanup comments) // Shamelessly copied from https://github.com/grafana/intro-to-mlt/blob/main/agent/config.river otelcol.receiver.otlp "otlp_receiver" { // We don't technically need this, but it shows how to change listen address and incoming port. From bf71def2f87a0870253965585e49dc7d65a34a6a Mon Sep 17 00:00:00 2001 From: Michel Hollands Date: Wed, 3 Apr 2024 14:12:53 +0100 Subject: [PATCH 4/4] create separate discovery.kubernetes for metrics Signed-off-by: Michel Hollands --- .../templates/agent/config.yaml | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/charts/meta-monitoring/templates/agent/config.yaml b/charts/meta-monitoring/templates/agent/config.yaml index 8f1df2b..d922d41 100644 --- a/charts/meta-monitoring/templates/agent/config.yaml +++ b/charts/meta-monitoring/templates/agent/config.yaml @@ -85,13 +85,40 @@ data: name = "metrics" } + discovery.kubernetes "metric_pods" { + role = "pod" + namespaces { + own_namespace = true + names = [ {{ include "agent.namespaces" . }} ] + } + } + discovery.relabel "only_http_metrics" { - targets = discovery.relabel.rename_meta_labels.output + targets = discovery.kubernetes.metric_pods.targets rule { - source_labels = ["__meta_kubernetes_pod_container_port_name"] - action = "keep" - regex = "http-metrics" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app_kubernetes_io_component"] + separator = "/" + regex = "(.*)/(.*)/(.*)" + replacement = "${1}/${2}-${3}" + target_label = "job" + } + rule { + target_label = "cluster" + replacement = "{{- .Values.clusterName -}}" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + action = "drop" + regex = "9095" } }