From d0580fd01f51fa73dbadbe38d13d20ba4b273e32 Mon Sep 17 00:00:00 2001 From: Andrew Seigner Date: Tue, 27 Mar 2018 14:34:10 -0700 Subject: [PATCH] Clean up Prometheus labels scraped from proxy The Prometheus scrape config collects from Conduit proxies, and maps Kubernetes labels to Prometheus labels, appending "k8s_". This change keeps the resultant Prometheus labels consistent with their source Kubernetes labels. For example: "deployment" and "pod_template_hash". Signed-off-by: Andrew Seigner --- cli/cmd/testdata/install_default.golden | 13 ++++++++---- cli/cmd/testdata/install_output.golden | 13 ++++++++---- cli/install/template.go | 13 ++++++++---- controller/script/simulate-proxy/main.go | 18 ++++++++-------- doc/prometheus.md | 13 ++++++++---- doc/proxy-metrics.md | 27 ++++++++++++------------ 6 files changed, 58 insertions(+), 39 deletions(-) diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index 56d00b55aee73..fe151968cc9e9 100755 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -569,18 +569,23 @@ data: - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace + # special case k8s' "job" label, to not interfere with prometheus' "job" + # label + # __meta_kubernetes_pod_label_conduit_io_proxy_job=foo => + # k8s_job=foo + - source_labels: [__meta_kubernetes_pod_label_conduit_io_proxy_job] + action: replace + target_label: k8s_job # __meta_kubernetes_pod_label_conduit_io_proxy_deployment=foo => - # k8s_deployment=foo + # deployment=foo - action: labelmap regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - replacement: k8s_$1 # drop all labels that we just made copies of in the previous labelmap - action: labeldrop regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - # __meta_kubernetes_pod_label_foo=bar => k8s_foo=bar + # __meta_kubernetes_pod_label_foo=bar => foo=bar - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - replacement: k8s_$1 ### Grafana ### --- diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 4678cef7fe4d2..d885108b92f74 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -572,18 +572,23 @@ data: - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace + # special case k8s' "job" label, to not interfere with prometheus' "job" + # label + # __meta_kubernetes_pod_label_conduit_io_proxy_job=foo => + # k8s_job=foo + - source_labels: [__meta_kubernetes_pod_label_conduit_io_proxy_job] + action: replace + target_label: k8s_job # __meta_kubernetes_pod_label_conduit_io_proxy_deployment=foo => - # k8s_deployment=foo + # deployment=foo - action: labelmap regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - replacement: k8s_$1 # drop all labels that we just made copies of in the previous labelmap - action: labeldrop regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - # __meta_kubernetes_pod_label_foo=bar => k8s_foo=bar + # __meta_kubernetes_pod_label_foo=bar => foo=bar - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - replacement: k8s_$1 ### Grafana ### --- diff --git a/cli/install/template.go b/cli/install/template.go index b8cc9b2eef37f..0a068e46a52af 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -384,18 +384,23 @@ data: - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace + # special case k8s' "job" label, to not interfere with prometheus' "job" + # label + # __meta_kubernetes_pod_label_conduit_io_proxy_job=foo => + # k8s_job=foo + - source_labels: [__meta_kubernetes_pod_label_conduit_io_proxy_job] + action: replace + target_label: k8s_job # __meta_kubernetes_pod_label_conduit_io_proxy_deployment=foo => - # k8s_deployment=foo + # deployment=foo - action: labelmap regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - replacement: k8s_$1 # drop all labels that we just made copies of in the previous labelmap - action: labeldrop regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - # __meta_kubernetes_pod_label_foo=bar => k8s_foo=bar + # __meta_kubernetes_pod_label_foo=bar => foo=bar - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - replacement: k8s_$1 ### Grafana ### --- diff --git a/controller/script/simulate-proxy/main.go b/controller/script/simulate-proxy/main.go index e820f8bed6135..d741182ec8535 100644 --- a/controller/script/simulate-proxy/main.go +++ b/controller/script/simulate-proxy/main.go @@ -44,13 +44,13 @@ var ( // for reference: https://github.com/runconduit/conduit/blob/master/doc/proxy-metrics.md#labels labels = []string{ // kubeResourceTypes - "k8s_daemon_set", - "k8s_deployment", + "daemon_set", + "deployment", "k8s_job", - "k8s_replication_controller", - "k8s_replica_set", + "replication_controller", + "replica_set", - "k8s_pod_template_hash", + "pod_template_hash", "namespace", // constantLabels @@ -196,10 +196,10 @@ func (s *simulatedProxy) generateProxyTraffic() { // newConduitLabel creates a label map to be used for metric generation. func (s *simulatedProxy) newConduitLabel(destinationPod string, isResponseLabel bool) prom.Labels { labelMap := prom.Labels{ - "direction": randomRequestDirection(), - "k8s_deployment": s.deploymentName, - "authority": "world.greeting:7778", - "namespace": s.namespace, + "direction": randomRequestDirection(), + "deployment": s.deploymentName, + "authority": "world.greeting:7778", + "namespace": s.namespace, } if labelMap["direction"] == "outbound" { labelMap["dst_deployment"] = destinationPod diff --git a/doc/prometheus.md b/doc/prometheus.md index afd776c842fcf..0ac991343a458 100755 --- a/doc/prometheus.md +++ b/doc/prometheus.md @@ -38,18 +38,23 @@ rich telemetry data to your cluster. Simply add the following item to your - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace + # special case k8s' "job" label, to not interfere with prometheus' "job" + # label + # __meta_kubernetes_pod_label_conduit_io_proxy_job=foo => + # k8s_job=foo + - source_labels: [__meta_kubernetes_pod_label_conduit_io_proxy_job] + action: replace + target_label: k8s_job # __meta_kubernetes_pod_label_conduit_io_proxy_deployment=foo => - # k8s_deployment=foo + # deployment=foo - action: labelmap regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - replacement: k8s_$1 # drop all labels that we just made copies of in the previous labelmap - action: labeldrop regex: __meta_kubernetes_pod_label_conduit_io_proxy_(.+) - # __meta_kubernetes_pod_label_foo=bar => k8s_foo=bar + # __meta_kubernetes_pod_label_foo=bar => foo=bar - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - replacement: k8s_$1 ``` That's it! Your Prometheus cluster is now configured to scrape Conduit's diff --git a/doc/proxy-metrics.md b/doc/proxy-metrics.md index d09504c0d0220..6992145085cdf 100644 --- a/doc/proxy-metrics.md +++ b/doc/proxy-metrics.md @@ -70,16 +70,15 @@ will correspond to `k8s_*` Prometheus labels: * `job`: The Prometheus job responsible for the collection, typically `conduit-proxy`. * `namespace`: Kubernetes namespace that the pod belongs to. -* `k8s_deployment`: The deployment that the pod belongs to (if applicable). +* `deployment`: The deployment that the pod belongs to (if applicable). * `k8s_job`: The job that the pod belongs to (if applicable). -* `k8s_replica_set`: The replica set that the pod belongs to (if applicable). -* `k8s_daemon_set`: The daemon set that the pod belongs to (if applicable). -* `k8s_replication_controller`: The replication controller that the pod belongs - to (if applicable). -* `k8s_pod_template_hash`: Corresponds to the - [pod-template-hash][pod-template-hash] Kubernetes - label. This value changes during redeploys and - rolling restarts. +* `replica_set`: The replica set that the pod belongs to (if applicable). +* `daemon_set`: The daemon set that the pod belongs to (if applicable). +* `replication_controller`: The replication controller that the pod belongs to + (if applicable). +* `pod_template_hash`: Corresponds to the [pod-template-hash][pod-template-hash] + Kubernetes label. This value changes during redeploys and + rolling restarts. Here's a concrete example, given the following pod snippet: @@ -98,11 +97,11 @@ The resulting Prometheus labels will look like this: ``` request_total{ namespace="emojivoto", - k8s_app="vote-bot", - k8s_conduit_io_control_plane_ns="conduit", - k8s_deployment="vote-bot", - k8s_pod_template_hash="3957278789", - k8s_test="vote-bot-test", + app="vote-bot", + conduit_io_control_plane_ns="conduit", + deployment="vote-bot", + pod_template_hash="3957278789", + test="vote-bot-test", instance="10.1.3.93:4191", job="conduit-proxy" }