From 073eebb16780a58fda5312a912d6feaa79d4588f Mon Sep 17 00:00:00 2001
From: Ramiro Algozino <ramiro@sighup.io>
Date: Wed, 17 Jul 2024 16:06:39 +0200
Subject: [PATCH] feature(onprem): enable haproxy monitoring

enable metrics scraping from the haproxy loadbalancers and deploy the
dashboards and alerts for haproxy from the monitoring module.
---
 .../monitoring/kustomization.yaml.tpl          |  4 ++++
 .../resources/haproxy-scrapeConfig.yaml.tpl    | 18 ++++++++++++++++++
 .../prometheus-agent/prometheus-agent.yaml.tpl |  3 +++
 .../distribution/scripts/pre-apply.sh.tpl      |  3 +++
 .../kubernetes/onpremises/haproxy.cfg.tpl      | 11 +++++++----
 5 files changed, 35 insertions(+), 4 deletions(-)
 create mode 100644 templates/distribution/manifests/monitoring/resources/haproxy-scrapeConfig.yaml.tpl

diff --git a/templates/distribution/manifests/monitoring/kustomization.yaml.tpl b/templates/distribution/manifests/monitoring/kustomization.yaml.tpl
index 887b031b..2754b1c3 100644
--- a/templates/distribution/manifests/monitoring/kustomization.yaml.tpl
+++ b/templates/distribution/manifests/monitoring/kustomization.yaml.tpl
@@ -19,6 +19,10 @@ resources:
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/blackbox-exporter" }}
 {{- if eq .spec.distribution.common.provider.type "none" }}{{/* none === on-premises */}}
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/kubeadm-sm" }}
+    {{- if .spec.kubernetes.loadBalancers.enabled }}
+  - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/haproxy" }}
+  - resources/haproxy-scrapeConfig.yaml
+    {{- end }}
 {{- end }}
 {{- if eq .spec.distribution.common.provider.type "eks" }}
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/eks-sm" }}
diff --git a/templates/distribution/manifests/monitoring/resources/haproxy-scrapeConfig.yaml.tpl b/templates/distribution/manifests/monitoring/resources/haproxy-scrapeConfig.yaml.tpl
new file mode 100644
index 00000000..e44033cb
--- /dev/null
+++ b/templates/distribution/manifests/monitoring/resources/haproxy-scrapeConfig.yaml.tpl
@@ -0,0 +1,18 @@
+{{- if and (eq .spec.distribution.common.provider.type "none") .spec.kubernetes.loadBalancers.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: haproxy-lb
+  namespace: monitoring
+  labels:
+    prometheus: k8s
+spec:
+  staticConfigs:
+    - labels:
+        job: prometheus
+      targets:
+        {{- range $lb := .spec.kubernetes.loadBalancers.hosts }}
+        - {{ $lb.ip }}:8405
+        {{- end }}
+{{- end }}
diff --git a/templates/distribution/manifests/monitoring/resources/prometheus-agent/prometheus-agent.yaml.tpl b/templates/distribution/manifests/monitoring/resources/prometheus-agent/prometheus-agent.yaml.tpl
index 0ddb0369..062a63c9 100644
--- a/templates/distribution/manifests/monitoring/resources/prometheus-agent/prometheus-agent.yaml.tpl
+++ b/templates/distribution/manifests/monitoring/resources/prometheus-agent/prometheus-agent.yaml.tpl
@@ -30,6 +30,9 @@ spec:
   probeSelector: {}
   serviceMonitorNamespaceSelector: {}
   serviceMonitorSelector: {}
+  scrapeConfigSelector:
+    matchLabels:
+      prometheus: k8s
 
   {{- $prometheusAgentArgs := dict "module" "monitoring" "package" "prometheusAgent" "spec" .spec }}
   tolerations:
diff --git a/templates/distribution/scripts/pre-apply.sh.tpl b/templates/distribution/scripts/pre-apply.sh.tpl
index 31ca1af6..655d11e2 100644
--- a/templates/distribution/scripts/pre-apply.sh.tpl
+++ b/templates/distribution/scripts/pre-apply.sh.tpl
@@ -384,6 +384,9 @@ deleteMonitoringCommon() {
   $kustomizebin build $vendorPath/modules/monitoring/katalog/kube-state-metrics | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
   $kustomizebin build $vendorPath/modules/monitoring/katalog/node-exporter | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
   $kustomizebin build $vendorPath/modules/monitoring/katalog/x509-exporter | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
+  {{- if and (eq .spec.distribution.common.provider.type "none") .spec.kubernetes.loadBalancers.enabled }}
+  $kustomizebin build $vendorPath/modules/monitoring/katalog/haproxy | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
+  {{- end }}
   echo "Monitoring common resources deleted."
 }
 
diff --git a/templates/kubernetes/onpremises/haproxy.cfg.tpl b/templates/kubernetes/onpremises/haproxy.cfg.tpl
index 5f1a54c2..82f84cf7 100644
--- a/templates/kubernetes/onpremises/haproxy.cfg.tpl
+++ b/templates/kubernetes/onpremises/haproxy.cfg.tpl
@@ -11,7 +11,7 @@ defaults
     timeout client 50s
     timeout server 50s
 
-listen  stats
+frontend stats
     bind *:1936
     mode            http
     log             global
@@ -19,9 +19,6 @@ listen  stats
     maxconn 10
 
     timeout client      100s
-    timeout server      100s
-    timeout connect     100s
-    timeout queue       100s
 
     stats enable
     stats uri /stats
@@ -30,6 +27,12 @@ listen  stats
     stats show-node
     stats auth {{ .spec.kubernetes.loadBalancers.stats.username }}:{{ .spec.kubernetes.loadBalancers.stats.password }}
 
+frontend prometheus
+  bind :8405
+  mode http
+  http-request use-service prometheus-exporter
+  no log
+
 frontend control-plane
     mode tcp
     bind *:6443 alpn h2,http/1.1