Fix tolerations for Kubernetes >= 1.24

The taints for control-plane Nodes are changed for cluster version >= 1.24. Add a new toleration for Pods running on control-plane Nodes to make sure they can be scheduled. Signed-off-by: Xu Liu <xliu2@vmware.com>
antrea-io · May 6, 2022 · 38523b7 · 38523b7
1 parent 2065919
commit 38523b7
Show file tree

Hide file tree

Showing 9 changed files with 30 additions and 18 deletions.
diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -203,6 +203,9 @@ controller:
     # Allow it to schedule onto master nodes.
     - key: node-role.kubernetes.io/master
       effect: NoSchedule
+    # Control-plane taint for Kubernetes >= 1.24.
+    - key: node-role.kubernetes.io/control-plane
+      effect: NoSchedule
   # -- Node selector for the antrea-controller Pod.
   nodeSelector:
     kubernetes.io/os: linux

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3723,6 +3723,8 @@ spec:
           operator: Exists
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       serviceAccountName: antrea-controller
       containers:
         - name: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3725,6 +3725,8 @@ spec:
           operator: Exists
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       serviceAccountName: antrea-controller
       containers:
         - name: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3723,6 +3723,8 @@ spec:
           operator: Exists
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       serviceAccountName: antrea-controller
       containers:
         - name: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3772,6 +3772,8 @@ spec:
           operator: Exists
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       serviceAccountName: antrea-controller
       containers:
         - name: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3723,6 +3723,8 @@ spec:
           operator: Exists
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       serviceAccountName: antrea-controller
       containers:
         - name: antrea-controller

diff --git a/test/e2e/framework.go b/test/e2e/framework.go
@@ -379,12 +379,19 @@ func labelNodeRoleControlPlane() string {
 	return labelNodeRoleControlPlane
 }
 
-func controlPlaneNoScheduleToleration() corev1.Toleration {
+func controlPlaneNoScheduleTolerations() []corev1.Toleration {
 	// the Node taint still uses "master" in K8s v1.20
-	return corev1.Toleration{
-		Key:      "node-role.kubernetes.io/master",
-		Operator: corev1.TolerationOpExists,
-		Effect:   corev1.TaintEffectNoSchedule,
+	return []corev1.Toleration{
+		{
+			Key:      "node-role.kubernetes.io/master",
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
+		{
+			Key:      "node-role.kubernetes.io/control-plane",
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
 	}
 }
 
@@ -1119,8 +1126,7 @@ func (data *TestData) CreatePodOnNodeInNamespace(name, ns string, nodeName, ctrN
 	}
 	if nodeName == controlPlaneNodeName() {
 		// tolerate NoSchedule taint if we want Pod to run on control-plane Node
-		noScheduleToleration := controlPlaneNoScheduleToleration()
-		podSpec.Tolerations = []corev1.Toleration{noScheduleToleration}
+		podSpec.Tolerations = controlPlaneNoScheduleTolerations()
 	}
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -2390,9 +2396,7 @@ func (data *TestData) createAgnhostPodOnNodeWithAnnotations(name string, ns stri
 
 func (data *TestData) createDaemonSet(name string, ns string, ctrName string, image string, cmd []string, args []string) (*appsv1.DaemonSet, func() error, error) {
 	podSpec := corev1.PodSpec{
-		Tolerations: []corev1.Toleration{
-			controlPlaneNoScheduleToleration(),
-		},
+		Tolerations: controlPlaneNoScheduleTolerations(),
 		Containers: []corev1.Container{
 			{
 				Name:            ctrName,
@@ -2464,9 +2468,7 @@ func (data *TestData) waitForDaemonSetPods(timeout time.Duration, dsName string,
 
 func (data *TestData) createStatefulSet(name string, ns string, size int32, ctrName string, image string, cmd []string, args []string, mutateFunc func(*appsv1.StatefulSet)) (*appsv1.StatefulSet, func() error, error) {
 	podSpec := corev1.PodSpec{
-		Tolerations: []corev1.Toleration{
-			controlPlaneNoScheduleToleration(),
-		},
+		Tolerations: controlPlaneNoScheduleTolerations(),
 		Containers: []corev1.Container{
 			{
 				Name:            ctrName,

diff --git a/test/e2e/networkpolicy_test.go b/test/e2e/networkpolicy_test.go
@@ -816,8 +816,7 @@ func testIngressPolicyWithEndPort(t *testing.T, data *TestData) {
 		}
 		if nodeName == controlPlaneNodeName() {
 			// tolerate NoSchedule taint if we want Pod to run on control-plane Node
-			noScheduleToleration := controlPlaneNoScheduleToleration()
-			podSpec.Tolerations = []corev1.Toleration{noScheduleToleration}
+			podSpec.Tolerations = controlPlaneNoScheduleTolerations()
 		}
 		pod := &corev1.Pod{
 			ObjectMeta: metav1.ObjectMeta{

diff --git a/test/e2e/performance_test.go b/test/e2e/performance_test.go
@@ -50,8 +50,6 @@ var (
 	customizePolicyRules = flag.Int("perf.http.policy_rules", 0, "Number of CIDRs in the network policy")
 	httpConcurrency      = flag.Int("perf.http.concurrency", 1, "Number of multiple requests to make at a time")
 	realizeTimeout       = flag.Duration("perf.realize.timeout", 5*time.Minute, "Timeout of the realization of network policies")
-	// tolerate NoSchedule taint to let the Pod run on control-plane Node
-	noScheduleToleration = controlPlaneNoScheduleToleration()
 	labelSelector        = &metav1.LabelSelector{
 		MatchLabels: map[string]string{"app": perfTestAppLabel},
 	}
@@ -118,7 +116,7 @@ func createPerfTestPodDefinition(name, containerName, image string) *corev1.Pod
 		"kubernetes.io/hostname": controlPlaneNodeName(),
 	}
 
-	podSpec.Tolerations = []corev1.Toleration{noScheduleToleration}
+	podSpec.Tolerations = controlPlaneNoScheduleTolerations()
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,