support external control plane

Signed-off-by: Zenghui Shi <zshi@redhat.com>
k8snetworkplumbingwg · Jul 1, 2022 · 39b8086 · 39b8086
1 parent 815fd13
commit 39b8086
Show file tree

Hide file tree

Showing 8 changed files with 574 additions and 9 deletions.
diff --git a/bindata/manifests/operator-webhook/server.yaml b/bindata/manifests/operator-webhook/server.yaml
@@ -29,15 +29,19 @@ spec:
         beta.kubernetes.io/os: linux
       affinity:
         nodeAffinity:
+          {{ if not .ExternalControlPlane }}
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
+              - key: node-role.kubernetes.io/worker
               - key: node-role.kubernetes.io/master
                 operator: Exists
             - matchExpressions:
                 - key: node-role.kubernetes.io/control-plane
                   operator: Exists
+          {{ end }}
       tolerations:
+      {{ if not .ExternalControlPlane }}
       - key: "node-role.kubernetes.io/master"
         operator: Exists
         effect: NoSchedule
@@ -47,6 +51,7 @@ spec:
       - key: "node.kubernetes.io/not-ready"
         operator: Exists
         effect: NoSchedule
+      {{ end }}
       containers:
       - name: webhook-server
         image: {{.SriovNetworkWebhookImage}}

diff --git a/bindata/manifests/webhook/server.yaml b/bindata/manifests/webhook/server.yaml
@@ -8,7 +8,7 @@ metadata:
   namespace: {{.Namespace}}
   annotations:
     kubernetes.io/description: |
-      This daemon set launches the network resource injector component on master nodes.
+      This daemon set launches the network resource injector component on master or worker nodes.
     release.openshift.io/version: "{{.ReleaseVersion}}"
 spec:
   selector:
@@ -31,16 +31,20 @@ spec:
       nodeSelector:
         beta.kubernetes.io/os: linux
       affinity:
+      {{ if not .ExternalControlPlane }}
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
+              - key: node-role.kubernetes.io/worker
               - key: node-role.kubernetes.io/master
                 operator: Exists
             - matchExpressions:
                 - key: node-role.kubernetes.io/control-plane
                   operator: Exists
+      {{ end }}
       tolerations:
+      {{ if not .ExternalControlPlane }}
       - key: "node-role.kubernetes.io/master"
         operator: Exists
         effect: NoSchedule
@@ -50,6 +54,7 @@ spec:
       - key: "node.kubernetes.io/not-ready"
         operator: Exists
         effect: NoSchedule
+      {{ end }}
       containers:
       - name: webhook-server
         image: {{.NetworkResourcesInjectorImage}}

diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go
@@ -226,6 +226,12 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(dc *sriovnetworkv1.Sriov
 		data.Data["ClusterType"] = constants.ClusterType
 		data.Data["CaBundle"] = os.Getenv("WEBHOOK_CA_BUNDLE")
 		data.Data["DevMode"] = os.Getenv("DEV_MODE")
+		external, err := constants.IsExternalControlPlaneCluster(r.Client)
+		if err != nil {
+			logger.Error(err, "Fail to get control plane topology")
+			return err
+		}
+		data.Data["ExternalControlPlane"] = external
 		objs, err := render.RenderDir(path, &data)
 		if err != nil {
 			logger.Error(err, "Fail to render webhook manifests")

diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -27,6 +27,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	. "github.com/onsi/gomega"
+	openshiftconfigv1 "github.com/openshift/api/config/v1"
 	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 
 	corev1 "k8s.io/api/core/v1"
@@ -80,6 +81,8 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).NotTo(HaveOccurred())
 	err = mcfgv1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
+	err = openshiftconfigv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
 
 	//+kubebuilder:scaffold:scheme
 
@@ -158,6 +161,17 @@ var _ = BeforeSuite(func(done Done) {
 	}
 	Expect(k8sClient.Create(context.TODO(), config)).Should(Succeed())
 
+	infra := &openshiftconfigv1.Infrastructure{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "cluster",
+		},
+		Spec: openshiftconfigv1.InfrastructureSpec{},
+		Status: openshiftconfigv1.InfrastructureStatus{
+			ControlPlaneTopology: openshiftconfigv1.HighlyAvailableTopologyMode,
+		},
+	}
+	Expect(k8sClient.Create(context.TODO(), infra)).Should(Succeed())
+
 	poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
 	poolConfig.SetNamespace(testNamespace)
 	poolConfig.SetName(constants.DefaultConfigName)

diff --git a/deploy/operator.yaml b/deploy/operator.yaml
@@ -79,6 +79,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: RELEASE_VERSION
               value: 4.3.0
             - name: SRIOV_CNI_BIN_PATH

diff --git a/deployment/sriov-network-operator/templates/operator.yaml b/deployment/sriov-network-operator/templates/operator.yaml
@@ -73,6 +73,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: RELEASE_VERSION
               value: {{ .Release.AppVersion }}
             - name: SRIOV_CNI_BIN_PATH

diff --git a/pkg/utils/cluster.go b/pkg/utils/cluster.go
@@ -15,39 +15,105 @@ import (
 
 const (
 	// default Infrastructure resource name for Openshift
-	infraResourceName = "cluster"
+	infraResourceName        = "cluster"
+	workerNodeLabelKey       = "node-role.kubernetes.io/worker"
+	masterNodeLabelKey       = "node-role.kubernetes.io/master"
+	controlPlaneNodeLabelKey = "node-role.kubernetes.io/control-plane"
 )
 
+func getNodeRole(node corev1.Node) string {
+	for k := range node.Labels {
+		if k == workerNodeLabelKey {
+			return "worker"
+		} else if k == masterNodeLabelKey || k == controlPlaneNodeLabelKey {
+			return "master"
+		}
+	}
+	return ""
+}
+
 func IsSingleNodeCluster(c client.Client) (bool, error) {
 	if os.Getenv("CLUSTER_TYPE") == ClusterTypeOpenshift {
-		return openshiftSingleNodeClusterStatus(c)
+		topo, err := openshiftControlPlaneTopologyStatus(c)
+		if err != nil {
+			return false, err
+		}
+		switch topo {
+		case configv1.SingleReplicaTopologyMode:
+			return true, nil
+		case configv1.HighlyAvailableTopologyMode:
+			return false, nil
+		}
+		return false, nil
 	}
 	return k8sSingleNodeClusterStatus(c)
 }
 
+// IsExternalControlPlaneCluster detects control plane location of the cluster.
+// On OpenShift, the control plane topology is configured in configv1.Infrastucture struct.
+// On kubernetes, it is determined by which node the sriov operator is scheduled on. If operator
+// pod is schedule on worker node, it is considered as external control plane.
+func IsExternalControlPlaneCluster(c client.Client) (bool, error) {
+	if os.Getenv("CLUSTER_TYPE") == ClusterTypeOpenshift {
+		topo, err := openshiftControlPlaneTopologyStatus(c)
+		if err != nil {
+			return false, err
+		}
+		switch topo {
+		case "External":
+			return true, nil
+		case configv1.HighlyAvailableTopologyMode:
+			return false, nil
+		case configv1.SingleReplicaTopologyMode:
+			return false, nil
+		}
+	} else {
+		role, err := operatorNodeRole(c)
+		if err != nil {
+			return false, err
+		}
+		if role == "worker" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func k8sSingleNodeClusterStatus(c client.Client) (bool, error) {
 	nodeList := &corev1.NodeList{}
 	err := c.List(context.TODO(), nodeList)
 	if err != nil {
-		glog.Errorf("IsSingleNodeCluster(): Failed to list nodes: %v", err)
+		glog.Errorf("k8sSingleNodeClusterStatus(): Failed to list nodes: %v", err)
 		return false, err
 	}
 
 	if len(nodeList.Items) == 1 {
-		glog.Infof("IsSingleNodeCluster(): one node found in the cluster")
+		glog.Infof("k8sSingleNodeClusterStatus(): one node found in the cluster")
 		return true, nil
 	}
 	return false, nil
 }
 
-func openshiftSingleNodeClusterStatus(c client.Client) (bool, error) {
+// operatorNodeRole returns role of the node where operator is scheduled on
+func operatorNodeRole(c client.Client) (string, error) {
+	node := corev1.Node{}
+	err := c.Get(context.TODO(), types.NamespacedName{Name: os.Getenv("NODE_NAME")}, &node)
+	if err != nil {
+		glog.Errorf("k8sIsExternalTopologyMode(): Failed to get node: %v", err)
+		return "", err
+	}
+
+	return getNodeRole(node), nil
+}
+
+func openshiftControlPlaneTopologyStatus(c client.Client) (configv1.TopologyMode, error) {
 	infra := &configv1.Infrastructure{}
 	err := c.Get(context.TODO(), types.NamespacedName{Name: infraResourceName}, infra)
 	if err != nil {
-		return false, err
+		return "", fmt.Errorf("openshiftControlPlaneTopologyStatus(): Failed to get Infrastructure (name: %s): %v", infraResourceName, err)
 	}
 	if infra == nil {
-		return false, fmt.Errorf("getting resource Infrastructure (name: %s) succeeded but object was nil", infraResourceName)
+		return "", fmt.Errorf("openshiftControlPlaneTopologyStatus(): getting resource Infrastructure (name: %s) succeeded but object was nil", infraResourceName)
 	}
-	return infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode, nil
+	return infra.Status.ControlPlaneTopology, nil
 }