Merge pull request #1352 from nathankot/haproxy

Add support for HAProxy
kube-hetzner · Jun 3, 2024 · 8c2179c · 8c2179c
2 parents 20d91cc + 3fe903e
commit 8c2179c
Show file tree

Hide file tree

Showing 7 changed files with 186 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ To achieve this, we built up on the shoulders of giants by choosing [openSUSE Mi
 - [x] Proper use of the **Hetzner private network** to minimize latency.
 - [x] Choose between **Flannel, Calico, or Cilium** as CNI.
 - [x] Optional **Wireguard** encryption of the Kube network for added security.
-- [x] **Traefik** or **Nginx** as ingress controller attached to a Hetzner load balancer with Proxy Protocol turned on.
+- [x] **Traefik**, **Nginx** or **HAProxy** as ingress controller attached to a Hetzner load balancer with Proxy Protocol turned on.
 - [x] **Automatic HA** with the default setting of three control-plane nodes and two agent nodes.
 - [x] **Autoscaling** nodes via the [kubernetes autoscaler](https://github.com/kubernetes/autoscaler).
 - [x] **Super-HA** with Nodepools for both control-plane and agent nodes that can be in different locations.
@@ -285,7 +285,7 @@ Most cluster components of Kube-Hetzner are deployed with the Rancher [Helm Char
 
 By default, we strive to give you optimal defaults, but if you wish, you can customize them.
 
-For Traefik, Nginx, Rancher, Cilium, Traefik, and Longhorn, for maximum flexibility, we give you the ability to configure them even better via helm values variables (e.g. `cilium_values`, see the advanced section in the kube.tf.example for more).
+For Traefik, Nginx, HAProxy, Rancher, Cilium, Traefik, and Longhorn, for maximum flexibility, we give you the ability to configure them even better via helm values variables (e.g. `cilium_values`, see the advanced section in the kube.tf.example for more).
 
 ## Adding Extras
 

diff --git a/init.tf b/init.tf
@@ -117,6 +117,7 @@ resource "null_resource" "kustomization" {
     helm_values_yaml = join("---\n", [
       local.traefik_values,
       local.nginx_values,
+      local.haproxy_values,
       local.calico_values,
       local.cilium_values,
       local.longhorn_values,
@@ -135,6 +136,7 @@ resource "null_resource" "kustomization" {
       coalesce(var.cilium_version, "N/A"),
       coalesce(var.traefik_version, "N/A"),
       coalesce(var.nginx_version, "N/A"),
+      coalesce(var.haproxy_version, "N/A"),
     ])
     options = join("\n", [
       for option, value in local.kured_options : "${option}=${value}"
@@ -179,6 +181,18 @@ resource "null_resource" "kustomization" {
     destination = "/var/post_install/nginx_ingress.yaml"
   }
 
+  # Upload haproxy ingress controller config
+  provisioner "file" {
+    content = templatefile(
+      "${path.module}/templates/haproxy_ingress.yaml.tpl",
+      {
+        version          = var.haproxy_version
+        values           = indent(4, trimspace(local.haproxy_values))
+        target_namespace = local.ingress_controller_namespace
+    })
+    destination = "/var/post_install/haproxy_ingress.yaml"
+  }
+
   # Upload the CCM patch config
   provisioner "file" {
     content = templatefile(

diff --git a/kube.tf.example b/kube.tf.example
@@ -399,7 +399,7 @@ module "kube-hetzner" {
   # ]
 
   # Enable delete protection on compatible resources to prevent accidental deletion from the Hetzner Cloud Console.
-  # This does not protect deletion from Terraform itself. 
+  # This does not protect deletion from Terraform itself.
   # enable_delete_protection = {
   #   floating_ip   = true
   #   load_balancer = true
@@ -463,13 +463,14 @@ module "kube-hetzner" {
   # If you want to specify the Kured version, set it below - otherwise it'll use the latest version available.
   # kured_version = ""
 
-  # If you want to enable the Nginx ingress controller (https://kubernetes.github.io/ingress-nginx/) instead of Traefik, you can set this to "nginx". Default is "traefik".
-  # By the default we load optimal Traefik and Nginx ingress controller config for Hetzner, however you may need to tweak it to your needs, so to do,
-  # we allow you to add a traefik_values and nginx_values, see towards the end of this file in the advanced section.
+  # Default is "traefik".
+  # If you want to enable the Nginx (https://kubernetes.github.io/ingress-nginx/) or HAProxy ingress controller instead of Traefik, you can set this to "nginx" or "haproxy".
+  # By the default we load optimal Traefik, Nginx or HAProxy ingress controller config for Hetzner, however you may need to tweak it to your needs, so to do,
+  # we allow you to add a traefik_values, nginx_values or haproxy_values, see towards the end of this file in the advanced section.
   # After the cluster is deployed, you can always use HelmChartConfig definition to tweak the configuration.
   # If you want to disable both controllers set this to "none"
   # ingress_controller = "nginx"
-  # Namespace in which to deploy the ingress controllers. Defaults to the ingress_controller variable, eg (nginx, traefik)
+  # Namespace in which to deploy the ingress controllers. Defaults to the ingress_controller variable, eg (haproxy, nginx, traefik)
   # ingress_target_namespace = ""
 
   # You can change the number of replicas for selected ingress controller here. The default 0 means autoselecting based on number of agent nodes (1 node = 1 replica, 2 nodes = 2 replicas, 3+ nodes = 3 replicas)
@@ -768,7 +769,7 @@ module "kube-hetzner" {
   #
   # Technical Note:
   # This setting sets the `load-balancer.hetzner.cloud/hostname` in the Hetzner LB definition, suitable for
-  # both Nginx and Traefik ingress controllers.
+  # HAProxy, Nginx and Traefik ingress controllers.
   #
   # Recommendation:
   # This setting is optional. If services communicate using direct service names, you can leave this unset.
@@ -986,6 +987,46 @@ controller:
       "load-balancer.hetzner.cloud/uses-proxyprotocol": "true"
   EOT */
 
+  # If you want to use a specific HAProxy helm chart version, set it below; otherwise, leave them as-is for the latest versions.
+  # haproxy_version = ""
+
+  # If you want to configure additional proxy protocol trusted IPs for haproxy, enter them here as a list of IPs (strings).
+  # Example for Cloudflare:
+  # haproxy_additional_proxy_protocol_ips = [
+  #   "173.245.48.0/20",
+  #   "103.21.244.0/22",
+  #   "103.22.200.0/22",
+  #   "103.31.4.0/22",
+  #   "141.101.64.0/18",
+  #   "108.162.192.0/18",
+  #   "190.93.240.0/20",
+  #   "188.114.96.0/20",
+  #   "197.234.240.0/22",
+  #   "198.41.128.0/17",
+  #   "162.158.0.0/15",
+  #   "104.16.0.0/13",
+  #   "104.24.0.0/14",
+  #   "172.64.0.0/13",
+  #   "131.0.72.0/22",
+  #   "2400:cb00::/32",
+  #   "2606:4700::/32",
+  #   "2803:f800::/32",
+  #   "2405:b500::/32",
+  #   "2405:8100::/32",
+  #   "2a06:98c0::/29",
+  #   "2c0f:f248::/32"
+  # ]
+
+  # Configure CPU and memory requests for each HAProxy pod
+  # haproxy_requests_cpu = "250m"
+  # haproxy_requests_memory = "400Mi"
+
+  # Override values given to the HAProxy helm chart.
+  # All HAProxy helm values can be found at https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/values.yaml
+  # Default values can be found at https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/blob/master/locals.tf
+  /*   haproxy_values = <<EOT
+  EOT */
+
   # Rancher, all Rancher helm values can be found at https://rancher.com/docs/rancher/v2.5/en/installation/install-rancher-on-k8s/chart-options/
   # The following is an example, please note that the current indentation inside the EOT is important.
   /*   rancher_values = <<EOT

diff --git a/locals.tf b/locals.tf
@@ -235,16 +235,19 @@ locals {
   ingress_controller_service_names = {
     "traefik" = "traefik"
     "nginx"   = "nginx-ingress-nginx-controller"
+    "haproxy" = "haproxy-kubernetes-ingress"
   }
 
   ingress_controller_install_resources = {
     "traefik" = ["traefik_ingress.yaml"]
     "nginx"   = ["nginx_ingress.yaml"]
+    "haproxy" = ["haproxy_ingress.yaml"]
   }
 
   default_ingress_namespace_mapping = {
     "traefik" = "traefik"
     "nginx"   = "nginx"
+    "haproxy" = "haproxy"
   }
 
   ingress_controller_namespace = var.ingress_target_namespace != "" ? var.ingress_target_namespace : lookup(local.default_ingress_namespace_mapping, var.ingress_controller, "")
@@ -570,7 +573,54 @@ controller:
 %{endif~}
   EOT
 
-  traefik_values = var.traefik_values != "" ? var.traefik_values : <<EOT
+  haproxy_values = var.haproxy_values != "" ? var.haproxy_values : <<EOT
+controller:
+  kind: "Deployment"
+  replicaCount: ${local.ingress_replica_count}
+  ingressClass: null
+  resources:
+    requests:
+      cpu: "${var.haproxy_requests_cpu}"
+      memory: "${var.haproxy_requests_memory}"
+  config:
+    ssl-redirect: "false"
+    forwarded-for: "true"
+%{if !local.using_klipper_lb~}
+    proxy-protocol: "${join(
+  ", ",
+  concat(
+    ["127.0.0.1/32", "10.0.0.0/8"],
+    var.haproxy_additional_proxy_protocol_ips
+  )
+)}"
+%{endif~}
+  service:
+    type: LoadBalancer
+    enablePorts:
+      quic: false
+      stat: false
+      prometheus: false
+%{if !local.using_klipper_lb~}
+    annotations:
+      "load-balancer.hetzner.cloud/name": "${local.load_balancer_name}"
+      "load-balancer.hetzner.cloud/use-private-ip": "true"
+      "load-balancer.hetzner.cloud/disable-private-ingress": "true"
+      "load-balancer.hetzner.cloud/disable-public-network": "${var.load_balancer_disable_public_network}"
+      "load-balancer.hetzner.cloud/ipv6-disabled": "${var.load_balancer_disable_ipv6}"
+      "load-balancer.hetzner.cloud/location": "${var.load_balancer_location}"
+      "load-balancer.hetzner.cloud/type": "${var.load_balancer_type}"
+      "load-balancer.hetzner.cloud/uses-proxyprotocol": "${!local.using_klipper_lb}"
+      "load-balancer.hetzner.cloud/algorithm-type": "${var.load_balancer_algorithm_type}"
+      "load-balancer.hetzner.cloud/health-check-interval": "${var.load_balancer_health_check_interval}"
+      "load-balancer.hetzner.cloud/health-check-timeout": "${var.load_balancer_health_check_timeout}"
+      "load-balancer.hetzner.cloud/health-check-retries": "${var.load_balancer_health_check_retries}"
+%{if var.lb_hostname != ""~}
+      "load-balancer.hetzner.cloud/hostname": "${var.lb_hostname}"
+%{endif~}
+%{endif~}
+  EOT
+
+traefik_values = var.traefik_values != "" ? var.traefik_values : <<EOT
 image:
   tag: ${var.traefik_image_tag}
 deployment:
@@ -687,7 +737,7 @@ autoscaling:
 %{endif~}
   EOT
 
-  rancher_values = var.rancher_values != "" ? var.rancher_values : <<EOT
+rancher_values = var.rancher_values != "" ? var.rancher_values : <<EOT
 hostname: "${var.rancher_hostname != "" ? var.rancher_hostname : var.lb_hostname}"
 replicas: ${length(local.control_plane_nodes)}
 bootstrapPassword: "${length(var.rancher_bootstrap_password) == 0 ? resource.random_password.rancher_bootstrap[0].result : var.rancher_bootstrap_password}"
@@ -697,19 +747,19 @@ global:
       enabled: false
   EOT
 
-  cert_manager_values = var.cert_manager_values != "" ? var.cert_manager_values : <<EOT
+cert_manager_values = var.cert_manager_values != "" ? var.cert_manager_values : <<EOT
 installCRDs: true
   EOT
 
-  kured_options = merge({
-    "reboot-command" : "/usr/bin/systemctl reboot",
-    "pre-reboot-node-labels" : "kured=rebooting",
-    "post-reboot-node-labels" : "kured=done",
-    "period" : "5m",
-    "reboot-sentinel" : "/sentinel/reboot-required"
-  }, var.kured_options)
+kured_options = merge({
+  "reboot-command" : "/usr/bin/systemctl reboot",
+  "pre-reboot-node-labels" : "kured=rebooting",
+  "post-reboot-node-labels" : "kured=done",
+  "period" : "5m",
+  "reboot-sentinel" : "/sentinel/reboot-required"
+}, var.kured_options)
 
-  k3s_registries_update_script = <<EOF
+k3s_registries_update_script = <<EOF
 DATE=`date +%Y-%m-%d_%H-%M-%S`
 if cmp -s /tmp/registries.yaml /etc/rancher/k3s/registries.yaml; then
   echo "No update required to the registries.yaml file"
@@ -729,7 +779,7 @@ else
 fi
 EOF
 
-  k3s_config_update_script = <<EOF
+k3s_config_update_script = <<EOF
 DATE=`date +%Y-%m-%d_%H-%M-%S`
 if cmp -s /tmp/config.yaml /etc/rancher/k3s/config.yaml; then
   echo "No update required to the config.yaml file"
@@ -751,7 +801,7 @@ else
 fi
 EOF
 
-  cloudinit_write_files_common = <<EOT
+cloudinit_write_files_common = <<EOT
 # Script to rename the private interface to eth1 and unify NetworkManager connection naming
 - path: /etc/cloud/rename_interface.sh
   content: |
@@ -916,7 +966,7 @@ EOF
 %{endif}
 EOT
 
-  cloudinit_runcmd_common = <<EOT
+cloudinit_runcmd_common = <<EOT
 # ensure that /var uses full available disk size, thanks to btrfs this is easy
 - [btrfs, 'filesystem', 'resize', 'max', '/var']
 
@@ -964,4 +1014,3 @@ EOT
 - [truncate, '-s', '0', '/var/log/audit/audit.log']
 EOT
 }
-
diff --git a/templates/haproxy_ingress.yaml.tpl b/templates/haproxy_ingress.yaml.tpl
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${target_namespace}
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: haproxy
+  namespace: kube-system
+spec:
+  chart: kubernetes-ingress
+  version: "${version}"
+  repo: https://haproxytech.github.io/helm-charts
+  targetNamespace: ${target_namespace}
+  bootstrap: true
+  valuesContent: |-
+    ${values}
diff --git a/values-export.tf b/values-export.tf
@@ -39,3 +39,10 @@ resource "local_file" "nginx_values" {
   filename        = "nginx_values.yaml"
   file_permission = "600"
 }
+
+resource "local_file" "haproxy_values" {
+  count           = var.export_values && var.ingress_controller == "haproxy" ? 1 : 0
+  content         = local.haproxy_values
+  filename        = "haproxy_values.yaml"
+  file_permission = "600"
+}
diff --git a/variables.tf b/variables.tf
@@ -382,8 +382,8 @@ variable "ingress_controller" {
   description = "The name of the ingress controller."
 
   validation {
-    condition     = contains(["traefik", "nginx", "none"], var.ingress_controller)
-    error_message = "Must be one of \"traefik\" or \"nginx\" or \"none\""
+    condition     = contains(["traefik", "nginx", "haproxy", "none"], var.ingress_controller)
+    error_message = "Must be one of \"traefik\" or \"nginx\" or \"haproxy\" or \"none\""
   }
 }
 
@@ -485,6 +485,36 @@ variable "nginx_values" {
   description = "Additional helm values file to pass to nginx as 'valuesContent' at the HelmChart."
 }
 
+variable "haproxy_requests_cpu" {
+  type        = string
+  default     = "250m"
+  description = "Setting for HAProxy controller.resources.requests.cpu"
+}
+
+variable "haproxy_requests_memory" {
+  type        = string
+  default     = "400Mi"
+  description = "Setting for HAProxy controller.resources.requests.memory"
+}
+
+variable "haproxy_additional_proxy_protocol_ips" {
+  type        = list(string)
+  default     = []
+  description = "Additional trusted proxy protocol IPs to pass to haproxy."
+}
+
+variable "haproxy_version" {
+  type        = string
+  default     = ""
+  description = "Version of HAProxy helm chart."
+}
+
+variable "haproxy_values" {
+  type        = string
+  default     = ""
+  description = "Helm values file to pass to haproxy as 'valuesContent' at the HelmChart, overriding the default."
+}
+
 variable "allow_scheduling_on_control_plane" {
   type        = bool
   default     = false
@@ -767,7 +797,7 @@ variable "rancher_hostname" {
 variable "lb_hostname" {
   type        = string
   default     = ""
-  description = "The Hetzner Load Balancer hostname, for either Traefik or Ingress-Nginx."
+  description = "The Hetzner Load Balancer hostname, for either Traefik, HAProxy or Ingress-Nginx."
 
   validation {
     condition     = can(regex("^(?:(?:(?:[A-Za-z0-9])|(?:[A-Za-z0-9](?:[A-Za-z0-9\\-]+)?[A-Za-z0-9]))+(\\.))+([A-Za-z]{2,})([\\/?])?([\\/?][A-Za-z0-9\\-%._~:\\/?#\\[\\]@!\\$&\\'\\(\\)\\*\\+,;=]+)?$", var.lb_hostname)) || var.lb_hostname == ""