From 7ae9316b8496d0fb7fc44513d26b2be340cfec5b Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Tue, 16 Feb 2021 14:11:50 -0500 Subject: [PATCH] Add support for metrics to gateways (#821) * Add support for metrics to gateways --- templates/ingress-gateways-deployment.yaml | 8 ++ templates/mesh-gateway-deployment.yaml | 8 ++ .../terminating-gateways-deployment.yaml | 8 ++ test/unit/ingress-gateways-deployment.bats | 101 +++++++++++++++++- test/unit/mesh-gateway-deployment.bats | 99 +++++++++++++++++ .../unit/terminating-gateways-deployment.bats | 99 +++++++++++++++++ values.yaml | 5 + 7 files changed, 327 insertions(+), 1 deletion(-) diff --git a/templates/ingress-gateways-deployment.yaml b/templates/ingress-gateways-deployment.yaml index b086a2723347..c3658cc4ac52 100644 --- a/templates/ingress-gateways-deployment.yaml +++ b/templates/ingress-gateways-deployment.yaml @@ -55,6 +55,11 @@ spec: ingress-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }} annotations: "consul.hashicorp.com/connect-inject": "false" + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + "prometheus.io/scrape": "true" + "prometheus.io/path": "/metrics" + "prometheus.io/port": "20200" + {{- end }} {{- if $defaults.annotations }} # We allow both default annotations and gateway-specific annotations {{- tpl $defaults.annotations $root | nindent 8 }} @@ -224,6 +229,9 @@ spec: } proxy { config { + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + envoy_prometheus_bind_addr = "${POD_IP}:20200" + {{- end }} envoy_gateway_no_default_bind = true envoy_gateway_bind_addresses { all-interfaces { diff --git a/templates/mesh-gateway-deployment.yaml b/templates/mesh-gateway-deployment.yaml index 10f066f24415..9fc5b812a884 100644 --- a/templates/mesh-gateway-deployment.yaml +++ b/templates/mesh-gateway-deployment.yaml @@ -35,6 +35,11 @@ spec: component: mesh-gateway annotations: "consul.hashicorp.com/connect-inject": "false" + {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }} + "prometheus.io/scrape": "true" + "prometheus.io/path": "/metrics" + "prometheus.io/port": "20200" + {{- end }} {{- if .Values.meshGateway.annotations }} {{- tpl .Values.meshGateway.annotations . | nindent 8 }} {{- end }} @@ -173,6 +178,9 @@ spec: consul-wan-federation = "1" } {{- end }} + {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }} + proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } } + {{- end }} port = {{ .Values.meshGateway.containerPort }} address = "${POD_IP}" tagged_addresses { diff --git a/templates/terminating-gateways-deployment.yaml b/templates/terminating-gateways-deployment.yaml index f48560eca694..f58205a875f5 100644 --- a/templates/terminating-gateways-deployment.yaml +++ b/templates/terminating-gateways-deployment.yaml @@ -53,6 +53,11 @@ spec: terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }} annotations: "consul.hashicorp.com/connect-inject": "false" + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + "prometheus.io/scrape": "true" + "prometheus.io/path": "/metrics" + "prometheus.io/port": "20200" + {{- end }} {{- if $defaults.annotations }} # We allow both default annotations and gateway-specific annotations {{- tpl $defaults.annotations $root | nindent 8 }} @@ -180,6 +185,9 @@ spec: {{- end }} address = "${POD_IP}" port = 8443 + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } } + {{- end }} checks = [ { name = "Terminating Gateway Listening" diff --git a/test/unit/ingress-gateways-deployment.bats b/test/unit/ingress-gateways-deployment.bats index c3087a22c7b7..a90938be1bbc 100644 --- a/test/unit/ingress-gateways-deployment.bats +++ b/test/unit/ingress-gateways-deployment.bats @@ -161,7 +161,7 @@ load _helpers [ "${actual}" = "key" ] } -@test "ingressGateway/Deployment: CA cert volume present when TLS is enabled" { +@test "ingressGateways/Deployment: CA cert volume present when TLS is enabled" { cd `chart_dir` local actual=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ @@ -258,6 +258,105 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# metrics + +@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "20200" ] +} + +@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "/metrics" ] +} + +@test "ingressGateways/Deployment: when global.metrics.enabled=true, sets proxy setting" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + + [ "${actual}" = "true" ] +} + +@test "ingressGateways/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + --set 'global.metrics.enableGatewayMetrics=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "ingressGateways/Deployment: when global.metrics.enabled=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ + --set 'ingressGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + #-------------------------------------------------------------------- # replicas diff --git a/test/unit/mesh-gateway-deployment.bats b/test/unit/mesh-gateway-deployment.bats index 14b348e7952b..3feb8f863c89 100755 --- a/test/unit/mesh-gateway-deployment.bats +++ b/test/unit/mesh-gateway-deployment.bats @@ -94,6 +94,105 @@ key2: value2' \ [ "${actual}" = "3" ] } +#-------------------------------------------------------------------- +# metrics + +@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "20200" ] +} + +@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "/metrics" ] +} + +@test "meshGateway/Deployment: when global.metrics.enabled=true, sets proxy setting" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + + [ "${actual}" = "true" ] +} + +@test "meshGateway/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + --set 'global.metrics.enableGatewayMetrics=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "meshGateway/Deployment: when global.metrics.enabled=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + #-------------------------------------------------------------------- # replicas diff --git a/test/unit/terminating-gateways-deployment.bats b/test/unit/terminating-gateways-deployment.bats index db00bbd5a89d..4ba18677df0c 100644 --- a/test/unit/terminating-gateways-deployment.bats +++ b/test/unit/terminating-gateways-deployment.bats @@ -258,6 +258,105 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# metrics + +@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "20200" ] +} + +@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "/metrics" ] +} + +@test "terminatingGateways/Deployment: when global.metrics.enabled=true, sets proxy setting" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + + [ "${actual}" = "true" ] +} + +@test "terminatingGateways/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=true' \ + --set 'global.metrics.enableGatewayMetrics=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "terminatingGateways/Deployment: when global.metrics.enabled=false, does not set proxy setting" { + cd `chart_dir` + local object=$(helm template \ + -s templates/terminating-gateways-deployment.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.metrics.enabled=false' \ + . | tee /dev/stderr | + yq '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + #-------------------------------------------------------------------- # replicas diff --git a/values.yaml b/values.yaml index b6277ea7f424..cf395c757cdf 100644 --- a/values.yaml +++ b/values.yaml @@ -245,6 +245,11 @@ global: # Only applicable if global.metrics.enabled is true. agentMetricsRetentionTime: 1m + # If true, mesh, terminating, and ingress gateways will expose their + # Envoy metrics on port 20200 at the /metrics path and all gateway pods + # will have Prometheus scrape annotations. Only applicable if global.metrics.enabled is true. + enableGatewayMetrics: true + # The consul sidecar ensures the Consul services # are always registered with their local Consul clients and is used by the # ingress/terminating/mesh gateways as well as with every Connect-injected service.