[v0.13.x] - Allow an AntiAffinty tolerant to help ensure even spread …

…of DNS pods (#1672) * Allow an AntiAffinty tolerant to help ensure even spread of DNS pods * Formatting * Indentation * Yaml fixes
kubernetes-retired · Jul 9, 2019 · e5a2165 · e5a2165
1 parent a5d440d
commit e5a2165
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 32 deletions.
diff --git a/builtin/files/cluster.yaml.tmpl b/builtin/files/cluster.yaml.tmpl
@@ -1341,6 +1341,9 @@ kubeDns:
   # When enabled, will deploy kube-dns to K8s controllers instead of workers.
   # deployToControllers: false
 
+  # When enabled, will attempt to deploy kube-dns to nodes in different AZs, useful when autoscaling or deploying to nodes.
+  # antiAffinityAvailabilityZone: false
+
   # DNS Autoscaler
   # Ref: https://github.com/kubernetes-incubator/cluster-proportional-autoscaler/
   autoscaler:

diff --git a/builtin/files/userdata/cloud-config-controller b/builtin/files/userdata/cloud-config-controller
@@ -4026,8 +4026,9 @@ write_files:
             spec:
               priorityClassName: system-node-critical
               serviceAccountName: coredns
-              {{ if .KubeDns.DeployToControllers -}}
+              {{ if or .KubeDns.DeployToControllers .KubeDns.AntiAffinityAvailabilityZone -}}
               affinity:
+                {{- if .KubeDns.DeployToControllers }}
                 nodeAffinity:
                   requiredDuringSchedulingIgnoredDuringExecution:
                     nodeSelectorTerms:
@@ -4036,7 +4037,19 @@ write_files:
                         operator: In
                         values:
                         - "master"
-
+                {{- end }}
+                {{- if .KubeDns.AntiAffinityAvailabilityZone }}
+                podAntiAffinity:
+                  preferredDuringSchedulingIgnoredDuringExecution:
+                    podAffinityTerm:
+                      labelSelector:
+                      - matchExpressions:
+                        - key: k8s-app
+                          operator: In
+                          values:
+                            - "kube-dns"
+                      topologyKey: "failure-domain.beta.kubernetes.io/zone"
+                {{- end }}
               tolerations:
                - key: "CriticalAddonsOnly"
                  operator: "Exists"

diff --git a/pkg/api/cluster.go b/pkg/api/cluster.go
@@ -161,10 +161,11 @@ func NewDefaultCluster() *Cluster {
 				IPVSMode: ipvsMode,
 			},
 			KubeDns: KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   false,
-				DeployToControllers: false,
-				TTL:                 30,
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          30,
 				Autoscaler: KubeDnsAutoscaler{
 					CoresPerReplica: 256,
 					NodesPerReplica: 16,

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -211,12 +211,13 @@ type KubeDnsAutoscaler struct {
 }
 
 type KubeDns struct {
-	Provider                 string            `yaml:"provider"`
-	NodeLocalResolver        bool              `yaml:"nodeLocalResolver"`
-	NodeLocalResolverOptions []string          `yaml:"nodeLocalResolverOptions"`
-	DeployToControllers      bool              `yaml:"deployToControllers"`
-	TTL                      int               `yaml:"ttl"`
-	Autoscaler               KubeDnsAutoscaler `yaml:"autoscaler"`
+	Provider                     string            `yaml:"provider"`
+	NodeLocalResolver            bool              `yaml:"nodeLocalResolver"`
+	NodeLocalResolverOptions     []string          `yaml:"nodeLocalResolverOptions"`
+	DeployToControllers          bool              `yaml:"deployToControllers"`
+	AntiAffinityAvailabilityZone bool              `yaml:"antiAffinityAvailabilityZone"`
+	TTL                          int               `yaml:"ttl"`
+	Autoscaler                   KubeDnsAutoscaler `yaml:"autoscaler"`
 }
 
 func (c *KubeDns) MergeIfEmpty(other KubeDns) {

diff --git a/pkg/model/cluster_test.go b/pkg/model/cluster_test.go
@@ -1119,10 +1119,11 @@ func TestKubeDns(t *testing.T) {
 			conf: `
 `,
 			kubeDns: api.KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   false,
-				DeployToControllers: false,
-				TTL:                 30,
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          30,
 				Autoscaler: api.KubeDnsAutoscaler{
 					CoresPerReplica: 256,
 					NodesPerReplica: 16,
@@ -1137,10 +1138,49 @@ kubeDns:
   deployToControllers: false
 `,
 			kubeDns: api.KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   false,
-				DeployToControllers: false,
-				TTL:                 30,
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          30,
+				Autoscaler: api.KubeDnsAutoscaler{
+					CoresPerReplica: 256,
+					NodesPerReplica: 16,
+					Min:             2,
+				},
+			},
+		},
+		{
+			conf: `
+kubeDns:
+  deployToControllers: false
+  antiAffinityAvailabilityZone: true
+`,
+			kubeDns: api.KubeDns{
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: true,
+				TTL:                          30,
+				Autoscaler: api.KubeDnsAutoscaler{
+					CoresPerReplica: 256,
+					NodesPerReplica: 16,
+					Min:             2,
+				},
+			},
+		},
+		{
+			conf: `
+kubeDns:
+  deployToControllers: true
+  antiAffinityAvailabilityZone: true
+`,
+			kubeDns: api.KubeDns{
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          true,
+				AntiAffinityAvailabilityZone: true,
+				TTL:                          30,
 				Autoscaler: api.KubeDnsAutoscaler{
 					CoresPerReplica: 256,
 					NodesPerReplica: 16,
@@ -1159,10 +1199,11 @@ kubeDns:
     min: 15
 `,
 			kubeDns: api.KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   true,
-				DeployToControllers: true,
-				TTL:                 30,
+				Provider:                     "coredns",
+				NodeLocalResolver:            true,
+				DeployToControllers:          true,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          30,
 				Autoscaler: api.KubeDnsAutoscaler{
 					CoresPerReplica: 5,
 					NodesPerReplica: 10,
@@ -1176,10 +1217,11 @@ kubeDns:
   provider: coredns
 `,
 			kubeDns: api.KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   false,
-				DeployToControllers: false,
-				TTL:                 30,
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          30,
 				Autoscaler: api.KubeDnsAutoscaler{
 					CoresPerReplica: 256,
 					NodesPerReplica: 16,
@@ -1194,10 +1236,11 @@ kubeDns:
   ttl: 5
 `,
 			kubeDns: api.KubeDns{
-				Provider:            "coredns",
-				NodeLocalResolver:   false,
-				DeployToControllers: false,
-				TTL:                 5,
+				Provider:                     "coredns",
+				NodeLocalResolver:            false,
+				DeployToControllers:          false,
+				AntiAffinityAvailabilityZone: false,
+				TTL:                          5,
 				Autoscaler: api.KubeDnsAutoscaler{
 					CoresPerReplica: 256,
 					NodesPerReplica: 16,