feature(onprem): enable haproxy monitoring

enable metrics scraping from the haproxy loadbalancers and deploy the
dashboards and alerts for haproxy from the monitoring module.

templates/distribution/manifests/monitoring/kustomization.yaml.tpl

@@ -19,6 +19,10 @@ resources:
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/blackbox-exporter" }}
 {{- if eq .spec.distribution.common.provider.type "none" }}{{/* none === on-premises */}}
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/kubeadm-sm" }}
+    {{- if .spec.kubernetes.loadBalancers.enabled }}
+  - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/haproxy" }}
+  - resources/haproxy-scrapeConfig.yaml
+    {{- end }}
 {{- end }}
 {{- if eq .spec.distribution.common.provider.type "eks" }}
   - {{ print "../" .spec.distribution.common.relativeVendorPath "/modules/monitoring/katalog/eks-sm" }}

templates/distribution/manifests/monitoring/resources/haproxy-scrapeConfig.yaml.tpl

@@ -0,0 +1,18 @@
+{{- if and (eq .spec.distribution.common.provider.type "none") .spec.kubernetes.loadBalancers.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: haproxy-lb
+  namespace: monitoring
+  labels:
+    prometheus: k8s
+spec:
+  staticConfigs:
+    - labels:
+        job: prometheus
+      targets:
+        {{- range $lb := .spec.kubernetes.loadBalancers.hosts }}
+        - {{ $lb.ip }}:8405
+        {{- end }}
+{{- end }}

templates/distribution/manifests/monitoring/resources/prometheus-agent/prometheus-agent.yaml.tpl

@@ -30,6 +30,9 @@ spec:
   probeSelector: {}
   serviceMonitorNamespaceSelector: {}
   serviceMonitorSelector: {}
+  scrapeConfigSelector:
+    matchLabels:
+      prometheus: k8s
 
   {{- $prometheusAgentArgs := dict "module" "monitoring" "package" "prometheusAgent" "spec" .spec }}
   tolerations:

templates/distribution/scripts/pre-apply.sh.tpl

@@ -384,6 +384,9 @@ deleteMonitoringCommon() {
   $kustomizebin build $vendorPath/modules/monitoring/katalog/kube-state-metrics | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
   $kustomizebin build $vendorPath/modules/monitoring/katalog/node-exporter | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
   $kustomizebin build $vendorPath/modules/monitoring/katalog/x509-exporter | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
+  {{- if and (eq .spec.distribution.common.provider.type "none") .spec.kubernetes.loadBalancers.enabled }}
+  $kustomizebin build $vendorPath/modules/monitoring/katalog/haproxy | $kubectlbin delete --ignore-not-found --wait --timeout=180s -f -
+  {{- end }}
   echo "Monitoring common resources deleted."
 }
 

templates/kubernetes/onpremises/haproxy.cfg.tpl

@@ -11,17 +11,14 @@ defaults
     timeout client 50s
     timeout server 50s
 
-listen  stats
+frontend stats
     bind *:1936
     mode            http
     log             global
 
     maxconn 10
 
     timeout client      100s
-    timeout server      100s
-    timeout connect     100s
-    timeout queue       100s
 
     stats enable
     stats uri /stats
@@ -30,6 +27,12 @@ listen  stats
     stats show-node
     stats auth {{ .spec.kubernetes.loadBalancers.stats.username }}:{{ .spec.kubernetes.loadBalancers.stats.password }}
 
+frontend prometheus
+  bind :8405
+  mode http
+  http-request use-service prometheus-exporter
+  no log
+
 frontend control-plane
     mode tcp
     bind *:6443 alpn h2,http/1.1