Set node affinity for more pods to ensure they run on general node po…

…ol (#2353)

Co-authored-by: Scott Blair <sblair@metrostar.com>

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -57,7 +57,6 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    #    "kubernetes.io/cluster/${var.name}"                       = "shared"
     "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
     propagate_at_launch                                       = true
   }, var.tags)
@@ -69,9 +68,36 @@ data "aws_eks_cluster_auth" "main" {
 
 resource "aws_eks_addon" "aws-ebs-csi-driver" {
   # required for Kubernetes v1.23+ on AWS
-  addon_name        = "aws-ebs-csi-driver"
-  cluster_name      = aws_eks_cluster.main.name
-  resolve_conflicts = "OVERWRITE"
+  addon_name                  = "aws-ebs-csi-driver"
+  cluster_name                = aws_eks_cluster.main.name
+  resolve_conflicts_on_create = "OVERWRITE"
+  resolve_conflicts_on_update = "OVERWRITE"
+
+  configuration_values = jsonencode({
+    controller = {
+      nodeSelector = {
+        "eks.amazonaws.com/nodegroup" = "general"
+      }
+    }
+  })
+
+  # Ensure cluster and node groups are created
+  depends_on = [
+    aws_eks_cluster.main,
+    aws_eks_node_group.main,
+  ]
+}
+
+resource "aws_eks_addon" "coredns" {
+  addon_name   = "coredns"
+  cluster_name = aws_eks_cluster.main.name
+
+  configuration_values = jsonencode({
+    nodeSelector = {
+      "eks.amazonaws.com/nodegroup" = "general"
+    }
+  })
+
   # Ensure cluster and node groups are created
   depends_on = [
     aws_eks_cluster.main,

src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf

@@ -3,7 +3,6 @@ resource "aws_vpc" "main" {
 
   enable_dns_support   = true
   enable_dns_hostnames = true
-  enable_classiclink   = false
 
   tags = merge({ Name = var.name }, var.tags, var.vpc_tags)
 }

src/_nebari/stages/infrastructure/template/aws/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.73.0"
+      version = "5.33.0"
     }
   }
   required_version = ">= 1.0"

src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf

@@ -19,6 +19,24 @@ resource "helm_release" "autoscaler" {
         clusterName = var.cluster-name
         enabled     = true
       }
+
+      affinity = {
+        nodeAffinity = {
+          requiredDuringSchedulingIgnoredDuringExecution = {
+            nodeSelectorTerms = [
+              {
+                matchExpressions = [
+                  {
+                    key      = "eks.amazonaws.com/nodegroup"
+                    operator = "In"
+                    values   = ["general"]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      }
     })
   ], var.overrides)
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf

@@ -557,6 +557,23 @@ resource "kubernetes_manifest" "deployment_admission_controller" {
               }
             },
           ]
+          affinity = {
+            nodeAffinity = {
+              requiredDuringSchedulingIgnoredDuringExecution = {
+                nodeSelectorTerms = [
+                  {
+                    matchExpressions = [
+                      {
+                        key      = var.node-group.key
+                        operator = "In"
+                        values   = [var.node-group.value]
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+          }
         }
       }
     }