Merge pull request #1012 from Silvest89/autoscaler-taints-labels

feat: taints and labels per autoscaler nodepool
kube-hetzner · Oct 23, 2023 · be65cd7 · be65cd7
2 parents 6404bcb + 62ac1a4
commit be65cd7
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 24 deletions.
diff --git a/autoscaler-agents.tf b/autoscaler-agents.tf
@@ -1,12 +1,28 @@
 locals {
   cluster_prefix = var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""
-  first_nodepool_snapshot_id = length(var.autoscaler_nodepools) == 0 ? "" : (
-    substr(var.autoscaler_nodepools[0].server_type, 0, 3) == "cax" ? data.hcloud_image.microos_arm_snapshot.id : data.hcloud_image.microos_x86_snapshot.id
-  )
+
+  imageList = {
+    arm64: tostring(data.hcloud_image.microos_arm_snapshot.id)
+    amd64: tostring(data.hcloud_image.microos_x86_snapshot.id)
+  }
+
+  nodeConfigName = var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""
+  cluster_config = {
+    imagesForArch: local.imageList
+    nodeConfigs: {
+      for index, nodePool in var.autoscaler_nodepools:
+      ("${local.nodeConfigName}${nodePool.name}") => {
+        cloudInit = data.cloudinit_config.autoscaler-config[index].rendered
+        labels = nodePool.labels
+        taints = nodePool.taints
+      }
+    }
+  }
+
+
   autoscaler_yaml = length(var.autoscaler_nodepools) == 0 ? "" : templatefile(
     "${path.module}/templates/autoscaler.yaml.tpl",
     {
-      cloudinit_config                    = base64encode(data.cloudinit_config.autoscaler-config[0].rendered)
       ca_image                            = var.cluster_autoscaler_image
       ca_version                          = var.cluster_autoscaler_version
       cluster_autoscaler_extra_args       = var.cluster_autoscaler_extra_args
@@ -15,7 +31,7 @@ locals {
       cluster_autoscaler_stderr_threshold = var.cluster_autoscaler_stderr_threshold
       ssh_key                             = local.hcloud_ssh_key_id
       ipv4_subnet_id                      = data.hcloud_network.k3s.id
-      snapshot_id                         = local.first_nodepool_snapshot_id
+      cluster_config                      = base64encode(jsonencode(local.cluster_config))
       firewall_id                         = hcloud_firewall.k3s.id
       cluster_name                        = local.cluster_prefix
       node_pools                          = var.autoscaler_nodepools
@@ -61,7 +77,7 @@ resource "null_resource" "configure_autoscaler" {
 }
 
 data "cloudinit_config" "autoscaler-config" {
-  count = length(var.autoscaler_nodepools) > 0 ? 1 : 0
+  count = length(var.autoscaler_nodepools)
 
   gzip          = true
   base64_encode = true
@@ -80,8 +96,8 @@ data "cloudinit_config" "autoscaler-config" {
           token         = local.k3s_token
           kubelet-arg   = local.kubelet_arg
           flannel-iface = local.flannel_iface
-          node-label    = concat(local.default_agent_labels, var.autoscaler_labels)
-          node-taint    = concat(local.default_agent_taints, var.autoscaler_taints)
+          node-label    = concat(local.default_agent_labels, [for k, v in var.autoscaler_nodepools[count.index].labels : "${k}=${v}"])
+          node-taint    = concat(local.default_agent_taints, [for taint in var.autoscaler_nodepools[count.index].taints : "${taint.key}=${taint.value}:${taint.effect}"])
           selinux       = true
         })
         install_k3s_agent_script     = join("\n", concat(local.install_k3s_agent, ["systemctl start k3s-agent"]))

diff --git a/kube.tf.example b/kube.tf.example
@@ -277,21 +277,18 @@ module "kube-hetzner" {
   #     location    = "fsn1"
   #     min_nodes   = 0
   #     max_nodes   = 5
+  #     labels      = {
+  #       "node.kubernetes.io/role=peak-workloads"
+  #     }
+  #     taints      = 
+  #     [{
+  #        key: "node.kubernetes.io/role"
+  #        value: "peak-workloads"
+  #        effect: "NoExecute"
+  #     }]  
   #   }
   # ]
 
-  # Add extra labels on nodes started by the Cluster Autoscaler
-  # This argument is not used if autoscaler_nodepools is not set, because the Cluster Autoscaler is installed only if autoscaler_nodepools is set
-  # autoscaler_labels = [
-  #   "node.kubernetes.io/role=peak-workloads"
-  # ]
-
-  # Add extra taints on nodes started by the Cluster Autoscaler
-  # This argument is not used if autoscaler_nodepools is not set, because the Cluster Autoscaler is installed only if autoscaler_nodepools is set
-  # autoscaler_taints = [
-  #   "node.kubernetes.io/role=specific-workloads:NoExecute"
-  # ]
-
   # Configuration of the Cluster Autoscaler binary
   #
   # These arguments and variables are not used if autoscaler_nodepools is not set, because the Cluster Autoscaler is installed only if autoscaler_nodepools is set.

diff --git a/templates/autoscaler.yaml.tpl b/templates/autoscaler.yaml.tpl
@@ -182,14 +182,12 @@ spec:
                 secretKeyRef:
                   name: hcloud
                   key: token
-          - name: HCLOUD_CLOUD_INIT
-            value: ${cloudinit_config}
+          - name: HCLOUD_CLUSTER_CONFIG
+            value: ${cluster_config}
           - name: HCLOUD_SSH_KEY
             value: '${ssh_key}'
           - name: HCLOUD_NETWORK
             value: '${ipv4_subnet_id}'
-          - name: HCLOUD_IMAGE
-            value: '${snapshot_id}'
           - name: HCLOUD_FIREWALL
             value: '${firewall_id}'
           volumeMounts:

diff --git a/variables.tf b/variables.tf
@@ -264,6 +264,12 @@ variable "autoscaler_nodepools" {
     location    = string
     min_nodes   = number
     max_nodes   = number
+    labels      = optional(map(string), {})
+    taints      = optional(list(object({
+      key       = string
+      value     = string
+      effect    = string
+    })), [])
   }))
   default = []
 }