Merge pull request #930 from M4t7e/cilium-refresh

Performance Improvements and Native Routing Mode for Cilium

README.md

@@ -372,17 +372,21 @@ cluster_ipv4_cidr = local.cluster_ipv4_cidr
 
 cilium_values = <<EOT
 ipam:
-  operator:
-    clusterPoolIPv4PodCIDRList:
-      - ${local.cluster_ipv4_cidr}
-kubeProxyReplacement: strict
-l7Proxy: "false"
+  mode: kubernetes
+k8s:
+  requireIPv4PodCIDR: true
+kubeProxyReplacement: true
+routingMode: native
+ipv4NativeRoutingCIDR: "10.0.0.0/8"
+endpointRoutes:
+  enabled: true
+loadBalancer:
+  acceleration: native
 bpf:
-  masquerade: "true"
+  masquerade: true
 egressGateway:
-  enabled: "true"
-extraConfig:
-  mtu: "1450"
+  enabled: true
+MTU: 1450
 EOT
 ```
 

init.tf

@@ -108,6 +108,7 @@ resource "null_resource" "kustomization" {
       coalesce(var.hetzner_csi_version, "N/A"),
       coalesce(var.kured_version, "N/A"),
       coalesce(var.calico_version, "N/A"),
+      coalesce(var.cilium_version, "N/A"),
     ])
     options = join("\n", [
       for option, value in var.kured_options : "${option}=${value}"
@@ -176,7 +177,8 @@ resource "null_resource" "kustomization" {
     content = templatefile(
       "${path.module}/templates/cilium.yaml.tpl",
       {
-        values = indent(4, trimspace(local.cilium_values))
+        values  = indent(4, trimspace(local.cilium_values))
+        version = var.cilium_version
     })
     destination = "/var/post_install/cilium.yaml"
   }

kube.tf.example

@@ -532,6 +532,18 @@ module "kube-hetzner" {
   # Also, see the cilium_values at towards the end of this file, in the advanced section.
   # cni_plugin = "cilium"
 
+  # You can choose the version of Cilium that you want. By default we keep the version up to date and configure Cilium with compatible settings according to the version.
+  # cilium_version = "v1.14.0"
+
+  # Set native-routing mode ("native") or tunneling mode ("tunnel"). Default: tunnel
+  # cilium_routing_mode = "native"
+
+  # Used when Cilium is configured in native routing mode. The CNI assumes that the underlying network stack will forward packets to this destination without the need to apply SNAT. Default: value of "cluster_ipv4_cidr"
+  # cilium_ipv4_native_routing_cidr = "10.0.0.0/8"
+
+  # Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. Default: false
+  # cilium_egress_gateway_enabled = true
+
   # You can choose the version of Calico that you want. By default, the latest is used.
   # More info on available versions can be found at https://github.com/projectcalico/calico/releases
   # Please note that if you are getting 403s from Github, it's also useful to set the version manually. However there is rarely a need for that!
@@ -662,20 +674,26 @@ module "kube-hetzner" {
   # We advise you to use the default values, and only change them if you know what you are doing!
 
   # Cilium, all Cilium helm values can be found at https://github.com/cilium/cilium/blob/master/install/kubernetes/cilium/values.yaml
+  # Be careful when maintaining your own cilium_values, as the choice of available settings depends on the Cilium version used. See also the cilium_version setting to fix a specific version.
   # The following is an example, please note that the current indentation inside the EOT is important.
   /*   cilium_values = <<EOT
 ipam:
   mode: kubernetes
-devices: "eth1"
 k8s:
   requireIPv4PodCIDR: true
-kubeProxyReplacement: strict
-l7Proxy: false
+kubeProxyReplacement: true
+routingMode: native
+ipv4NativeRoutingCIDR: "10.0.0.0/8"
+endpointRoutes:
+  enabled: true
+loadBalancer:
+  acceleration: native
+bpf:
+  masquerade: true
 encryption:
   enabled: true
   type: wireguard
-extraConfig:
-  mtu: "1450"
+MTU: 1450
   EOT */
 
   # Cert manager, all cert-manager helm values can be found at https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml

locals.tf

@@ -12,6 +12,8 @@ locals {
   kured_version  = var.kured_version != null ? var.kured_version : data.github_release.kured[0].release_tag
   calico_version = length(data.github_release.calico) == 0 ? var.calico_version : data.github_release.calico[0].release_tag
 
+  cilium_ipv4_native_routing_cidr = coalesce(var.cilium_ipv4_native_routing_cidr, var.cluster_ipv4_cidr)
+
   additional_k3s_environment = join("\n",
     [
       for var_name, var_value in var.additional_k3s_environment :
@@ -349,19 +351,43 @@ locals {
   }
 
   cilium_values = var.cilium_values != "" ? var.cilium_values : <<EOT
+# Enable Kubernetes host-scope IPAM mode (required for K3s + Hetzner CCM)
 ipam:
- operator:
-  clusterPoolIPv4PodCIDRList:
-   - ${var.cluster_ipv4_cidr}
-devices: "eth1"
-%{if var.enable_wireguard~}
-l7Proxy: false
+  mode: kubernetes
+k8s:
+  requireIPv4PodCIDR: true
+
+# Replace kube-proxy with Cilium
+kubeProxyReplacement: true
+
+# Set Tunnel Mode or Native Routing Mode (supported by Hetzner CCM Route Controller)
+routingMode: "${var.cilium_routing_mode}"
+%{if var.cilium_routing_mode == "native"~}
+ipv4NativeRoutingCIDR: "${local.cilium_ipv4_native_routing_cidr}"
+%{endif~}
+
+endpointRoutes:
+  # Enable use of per endpoint routes instead of routing via the cilium_host interface.
+  enabled: true
+
+loadBalancer:
+  # Enable LoadBalancer & NodePort XDP Acceleration (direct routing (routingMode=native) is recommended to achieve optimal performance)
+  acceleration: native
+
+bpf:
+  # Enable eBPF-based Masquerading ("The eBPF-based implementation is the most efficient implementation")
+  masquerade: true
+%{if var.enable_wireguard}
 encryption:
   enabled: true
   type: wireguard
 %{endif~}
-extraConfig:
-  mtu: "1450"
+%{if var.cilium_egress_gateway_enabled}
+egressGateway:
+  enabled: true
+%{endif~}
+
+MTU: 1450
   EOT
 
   # Not to be confused with the other helm values, this is used for the calico.yaml kustomize patch

templates/cilium.yaml.tpl

@@ -7,6 +7,7 @@ metadata:
 spec:
   chart: cilium
   repo: https://helm.cilium.io/
+  version: "${version}"
   targetNamespace: kube-system
   bootstrap: true
   valuesContent: |-

variables.tf

@@ -454,12 +454,41 @@ variable "cni_plugin" {
   }
 }
 
+variable "cilium_egress_gateway_enabled" {
+  type        = bool
+  default     = false
+  description = "Enables egress gateway to redirect and SNAT the traffic that leaves the cluster."
+}
+
+variable "cilium_ipv4_native_routing_cidr" {
+  type        = string
+  default     = null
+  description = "Used when Cilium is configured in native routing mode. The CNI assumes that the underlying network stack will forward packets to this destination without the need to apply SNAT. Default: value of \"cluster_ipv4_cidr\""
+}
+
+variable "cilium_routing_mode" {
+  type        = string
+  default     = "tunnel"
+  description = "Set native-routing mode (\"native\") or tunneling mode (\"tunnel\")."
+
+  validation {
+    condition     = contains(["tunnel", "native"], var.cilium_routing_mode)
+    error_message = "The cilium_routing_mode must be one of \"tunnel\" or \"native\"."
+  }
+}
+
 variable "cilium_values" {
   type        = string
   default     = ""
   description = "Additional helm values file to pass to Cilium as 'valuesContent' at the HelmChart."
 }
 
+variable "cilium_version" {
+  type        = string
+  default     = "v1.14.0"
+  description = "Version of Cilium."
+}
+
 variable "calico_values" {
   type        = string
   default     = ""