Add support for metrics to gateways (hashicorp#821)

* Add support for metrics to gateways
duckhan · Mar 10, 2021 · 7ae9316 · 7ae9316
1 parent 1d4122a
commit 7ae9316
Show file tree

Hide file tree

Showing 7 changed files with 327 additions and 1 deletion.
diff --git a/templates/ingress-gateways-deployment.yaml b/templates/ingress-gateways-deployment.yaml
@@ -55,6 +55,11 @@ spec:
         ingress-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+        "prometheus.io/scrape": "true"
+        "prometheus.io/path": "/metrics"
+        "prometheus.io/port": "20200"
+        {{- end }}
         {{- if $defaults.annotations }}
         # We allow both default annotations and gateway-specific annotations
         {{- tpl $defaults.annotations $root | nindent 8 }}
@@ -224,6 +229,9 @@ spec:
                   }
                   proxy {
                     config {
+                      {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+                      envoy_prometheus_bind_addr = "${POD_IP}:20200"
+                      {{- end }}
                       envoy_gateway_no_default_bind = true
                       envoy_gateway_bind_addresses {
                         all-interfaces {

diff --git a/templates/mesh-gateway-deployment.yaml b/templates/mesh-gateway-deployment.yaml
@@ -35,6 +35,11 @@ spec:
         component: mesh-gateway
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }}
+        "prometheus.io/scrape": "true"
+        "prometheus.io/path": "/metrics"
+        "prometheus.io/port": "20200"
+        {{- end }}
         {{- if .Values.meshGateway.annotations }}
         {{- tpl .Values.meshGateway.annotations . | nindent 8 }}
         {{- end }}
@@ -173,6 +178,9 @@ spec:
                     consul-wan-federation = "1"
                   }
                   {{- end }}
+                  {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }}
+                  proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } }
+                  {{- end }}
                   port = {{ .Values.meshGateway.containerPort }}
                   address = "${POD_IP}"
                   tagged_addresses {

diff --git a/templates/terminating-gateways-deployment.yaml b/templates/terminating-gateways-deployment.yaml
@@ -53,6 +53,11 @@ spec:
         terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+        "prometheus.io/scrape": "true"
+        "prometheus.io/path": "/metrics"
+        "prometheus.io/port": "20200"
+        {{- end }}
         {{- if $defaults.annotations }}
         # We allow both default annotations and gateway-specific annotations
         {{- tpl $defaults.annotations $root | nindent 8 }}
@@ -180,6 +185,9 @@ spec:
                   {{- end }}
                   address = "${POD_IP}"
                   port = 8443
+                  {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+                  proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } }
+                  {{- end }}
                   checks = [
                     {
                       name = "Terminating Gateway Listening"

diff --git a/test/unit/ingress-gateways-deployment.bats b/test/unit/ingress-gateways-deployment.bats
@@ -161,7 +161,7 @@ load _helpers
   [ "${actual}" = "key" ]
 }
 
-@test "ingressGateway/Deployment: CA cert volume present when TLS is enabled" {
+@test "ingressGateways/Deployment: CA cert volume present when TLS is enabled" {
   cd `chart_dir`
   local actual=$(helm template \
       -s templates/ingress-gateways-deployment.yaml  \
@@ -258,6 +258,105 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+#--------------------------------------------------------------------
+# metrics
+
+@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "20200" ]
+}
+
+@test "ingressGateways/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "/metrics" ]
+}
+
+@test "ingressGateways/Deployment: when global.metrics.enabled=true, sets proxy setting" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+
+  [ "${actual}" = "true" ]
+}
+
+@test "ingressGateways/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      --set 'global.metrics.enableGatewayMetrics=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "ingressGateways/Deployment: when global.metrics.enabled=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
 #--------------------------------------------------------------------
 # replicas
 

diff --git a/test/unit/mesh-gateway-deployment.bats b/test/unit/mesh-gateway-deployment.bats
@@ -94,6 +94,105 @@ key2: value2' \
   [ "${actual}" = "3" ]
 }
 
+#--------------------------------------------------------------------
+# metrics
+
+@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "20200" ]
+}
+
+@test "meshGateway/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "/metrics" ]
+}
+
+@test "meshGateway/Deployment: when global.metrics.enabled=true, sets proxy setting" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+
+  [ "${actual}" = "true" ]
+}
+
+@test "meshGateway/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      --set 'global.metrics.enableGatewayMetrics=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "meshGateway/Deployment: when global.metrics.enabled=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
 #--------------------------------------------------------------------
 # replicas
 

diff --git a/test/unit/terminating-gateways-deployment.bats b/test/unit/terminating-gateways-deployment.bats
@@ -258,6 +258,105 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+#--------------------------------------------------------------------
+# metrics
+
+@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus scrape=true annotations" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus port=20200 annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "20200" ]
+}
+
+@test "terminatingGateways/Deployment: when global.metrics.enabled=true, adds prometheus path=/metrics annotation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+       yq -s -r '.[0].spec.template.metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "/metrics" ]
+}
+
+@test "terminatingGateways/Deployment: when global.metrics.enabled=true, sets proxy setting" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+
+  [ "${actual}" = "true" ]
+}
+
+@test "terminatingGateways/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=true'  \
+      --set 'global.metrics.enableGatewayMetrics=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "terminatingGateways/Deployment: when global.metrics.enabled=false, does not set proxy setting" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.metrics.enabled=false'  \
+      . | tee /dev/stderr |
+      yq '.spec.template' | tee /dev/stderr)
+
+  local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/port"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/scrape"' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
 #--------------------------------------------------------------------
 # replicas