diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml index 4298f4373f..3463b60726 100644 --- a/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml @@ -4,6 +4,8 @@ apiVersion: apps/v1 metadata: {{ if eq .OVN_NODE_MODE "dpu-host" }} name: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + name: ovnkube-node-smart-nic {{ else }} name: ovnkube-node {{ end }} @@ -17,6 +19,8 @@ spec: matchLabels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + app: ovnkube-node-smart-nic {{ else }} app: ovnkube-node {{ end }} @@ -31,6 +35,8 @@ spec: labels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + app: ovnkube-node-smart-nic {{ else }} app: ovnkube-node {{ end }} @@ -44,14 +50,30 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: network.operator.openshift.io/dpu-host + {{ if .DpuHostModeLabel }} + - key: {{ .DpuHostModeLabel }} {{ if eq .OVN_NODE_MODE "dpu-host" }} operator: Exists + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + operator: DoesNotExist {{ else }} operator: DoesNotExist {{ end }} - - key: network.operator.openshift.io/dpu + {{ end }} + {{ if .SmartNicModeLabel }} + - key: {{ .SmartNicModeLabel }} + {{ if eq .OVN_NODE_MODE "dpu-host" }} + operator: DoesNotExist + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + operator: Exists + {{ else }} operator: DoesNotExist + {{ end }} + {{ end }} + {{ if .DpuModeLabel }} + - key: {{ .DpuModeLabel }} + operator: DoesNotExist + {{ end }} serviceAccountName: ovn-kubernetes-node hostNetwork: true dnsPolicy: Default @@ -69,7 +91,7 @@ spec: {{end}} initContainers: # ovnkube-node-init: wait for sbdb ready - {{ if eq .OVN_NODE_MODE "full" }} + {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} - name: ovnkube-node-init image: "{{.OvnImage}}" command: @@ -122,7 +144,7 @@ spec: # /run/openvswitch -> tmpfs - ovsdb sockets # /env -> configmap env-overrides - debug overrides containers: - {{ if eq .OVN_NODE_MODE "full" }} + {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} {{if .ENABLE_OVN_NODE_PROXY}} # ovnkube-node-proxy redirects ovn sbdb traffic to http proxy - name: ovnkube-node-proxy @@ -430,6 +452,9 @@ spec: if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" fi + if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then + node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" + fi multi_network_enabled_flag= if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then @@ -509,6 +534,10 @@ spec: - name: IPFIX_SAMPLING value: "{{.IPFIXSampling}}" {{ end }} + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + - name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME + value: {{ .MgmtPortResourceName }} + {{ end }} - name: K8S_NODE valueFrom: fieldRef: @@ -573,6 +602,13 @@ spec: requests: cpu: 10m memory: 300Mi + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + {{ .MgmtPortResourceName }}: '1' + {{ end }} + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + limits: + {{ .MgmtPortResourceName }}: '1' + {{ end }} lifecycle: preStop: exec: @@ -676,7 +712,7 @@ spec: - name: run-ovn hostPath: path: /var/run/ovn - {{ if eq .OVN_NODE_MODE "full" }} + {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} # Used for placement of ACL audit logs - name: node-log hostPath: diff --git a/bindata/network/ovn-kubernetes/self-hosted/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/self-hosted/ovnkube-node.yaml index 0db713d6d1..ef34251a3a 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/ovnkube-node.yaml @@ -4,6 +4,8 @@ apiVersion: apps/v1 metadata: {{ if eq .OVN_NODE_MODE "dpu-host" }} name: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + name: ovnkube-node-smart-nic {{ else }} name: ovnkube-node {{ end }} @@ -17,6 +19,8 @@ spec: matchLabels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + app: ovnkube-node-smart-nic {{ else }} app: ovnkube-node {{ end }} @@ -31,6 +35,8 @@ spec: labels: {{ if eq .OVN_NODE_MODE "dpu-host" }} app: ovnkube-node-dpu-host + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + app: ovnkube-node-smart-nic {{ else }} app: ovnkube-node {{ end }} @@ -44,14 +50,30 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: network.operator.openshift.io/dpu-host + {{ if .DpuHostModeLabel }} + - key: {{ .DpuHostModeLabel }} {{ if eq .OVN_NODE_MODE "dpu-host" }} operator: Exists + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + operator: DoesNotExist {{ else }} operator: DoesNotExist {{ end }} - - key: network.operator.openshift.io/dpu + {{ end }} + {{ if .SmartNicModeLabel }} + - key: {{ .SmartNicModeLabel }} + {{ if eq .OVN_NODE_MODE "dpu-host" }} + operator: DoesNotExist + {{ else if eq .OVN_NODE_MODE "smart-nic" }} + operator: Exists + {{ else }} operator: DoesNotExist + {{ end }} + {{ end }} + {{ if .DpuModeLabel }} + - key: {{ .DpuModeLabel }} + operator: DoesNotExist + {{ end }} serviceAccountName: ovn-kubernetes-node hostNetwork: true dnsPolicy: Default @@ -64,7 +86,7 @@ spec: # /run/openvswitch -> tmpfs - ovsdb sockets # /env -> configmap env-overrides - debug overrides containers: - {{ if eq .OVN_NODE_MODE "full" }} + {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} # ovn-controller: programs the vswitch with flows from the sbdb - name: ovn-controller image: "{{.OvnImage}}" @@ -77,8 +99,8 @@ spec: set -o allexport source "/env/${K8S_NODE}" set +o allexport - fi - + fi + echo "$(date -Iseconds) - starting ovn-controller" exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ @@ -134,7 +156,7 @@ spec: set -euo pipefail # Rotate audit log files when then get to max size (in bytes) - MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) + MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) LOGFILE=/var/log/ovn/acl-audit-log.log CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) @@ -143,14 +165,14 @@ spec: while true do - # Make sure ovn-controller's logfile exists, and get current size in bytes - if [ -f "$LOGFILE" ]; then + # Make sure ovn-controller's logfile exists, and get current size in bytes + if [ -f "$LOGFILE" ]; then file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - else + else ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1` - fi - + fi + if [ $file_size -gt $MAXFILESIZE ];then echo "Rotating OVN ACL Log File" timestamp=`date '+%Y-%m-%dT%H-%M-%S'` @@ -159,8 +181,8 @@ spec: CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid) fi - # sleep for 30 seconds to avoid wasting CPU - sleep 30 + # sleep for 30 seconds to avoid wasting CPU + sleep 30 done resources: requests: @@ -190,7 +212,7 @@ spec: TS=$(date +%s) WARN_TS=$(( ${TS} + $(( 20 * 60)) )) HAS_LOGGED_INFO=0 - + log_missing_certs(){ CUR_TS=$(date +%s) if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then @@ -204,7 +226,7 @@ spec: log_missing_certs sleep 5 done - + echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy exec /usr/bin/kube-rbac-proxy \ --logtostderr \ @@ -337,6 +359,9 @@ spec: if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" fi + if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then + node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" + fi multi_network_enabled_flag= if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then @@ -404,6 +429,10 @@ spec: - name: IPFIX_SAMPLING value: "{{.IPFIXSampling}}" {{ end }} + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + - name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME + value: {{ .MgmtPortResourceName }} + {{ end }} - name: K8S_NODE valueFrom: fieldRef: @@ -468,6 +497,13 @@ spec: requests: cpu: 10m memory: 300Mi + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + {{ .MgmtPortResourceName }}: '1' + {{ end }} + {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} + limits: + {{ .MgmtPortResourceName }}: '1' + {{ end }} lifecycle: preStop: exec: @@ -571,13 +607,13 @@ spec: - name: run-ovn hostPath: path: /var/run/ovn - {{ if eq .OVN_NODE_MODE "full" }} - # Used for placement of ACL audit logs + {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} + # Used for placement of ACL audit logs - name: node-log - hostPath: + hostPath: path: /var/log/ovn - name: log-socket - hostPath: + hostPath: path: /dev/log {{ end }} # For CNI server diff --git a/hack/dpu-mode.yaml b/hack/dpu-mode.yaml deleted file mode 100644 index 131d1c4d97..0000000000 --- a/hack/dpu-mode.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Example ConfigMap to enable dpu-host mode with OVNKubernetes -apiVersion: v1 -kind: ConfigMap -metadata: - name: dpu-mode-config - namespace: openshift-network-operator -data: - mode: "dpu-host" -immutable: true diff --git a/hack/hardware-offload-config.yaml b/hack/hardware-offload-config.yaml new file mode 100644 index 0000000000..c500d98eac --- /dev/null +++ b/hack/hardware-offload-config.yaml @@ -0,0 +1,11 @@ +# Example ConfigMap to watch for specific label names and resource names for hardware offloading with OVNKubernetes +apiVersion: v1 +kind: ConfigMap +metadata: + name: hardware-offload-config + namespace: openshift-network-operator +data: + dpu-host-mode-label: "network.operator.openshift.io/dpu-host" + dpu-mode-label: "network.operator.openshift.io/dpu" + smart-nic-mode-label: "network.operator.openshift.io/smart-nic" + mgmt-port-resource-name: "openshift.io/mgmtvf" diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go index 0c3f16f064..d6b0619ba5 100644 --- a/pkg/bootstrap/types.go +++ b/pkg/bootstrap/types.go @@ -39,9 +39,15 @@ type OVNHyperShiftBootstrapResult struct { type OVNConfigBoostrapResult struct { GatewayMode string - NodeMode string HyperShiftConfig *OVNHyperShiftBootstrapResult DisableUDPAggregation bool + DpuHostModeLabel string + DpuHostModeNodes []string + DpuModeLabel string + DpuModeNodes []string + SmartNicModeLabel string + SmartNicModeNodes []string + MgmtPortResourceName string } // OVNUpdateStatus contains the status of existing daemonset diff --git a/pkg/controller/operconfig/operconfig_controller.go b/pkg/controller/operconfig/operconfig_controller.go index b49f55f195..0dc5a7a2da 100644 --- a/pkg/controller/operconfig/operconfig_controller.go +++ b/pkg/controller/operconfig/operconfig_controller.go @@ -117,13 +117,15 @@ func add(mgr manager.Manager, r *ReconcileOperConfig) error { return err } - // Watch when nodes are created too - newNodePredicate := predicate.Funcs{ + // Watch when nodes are created and updated. + // We need to watch when nodes are updated since we are interested in the labels + // of nodes for hardware offloading. + nodePredicate := predicate.Funcs{ CreateFunc: func(_ event.CreateEvent) bool { return true }, UpdateFunc: func(_ event.UpdateEvent) bool { - return false + return true }, DeleteFunc: func(_ event.DeleteEvent) bool { return true @@ -132,7 +134,7 @@ func add(mgr manager.Manager, r *ReconcileOperConfig) error { if err := c.Watch( &source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(reconcileOperConfig), - newNodePredicate, + nodePredicate, ); err != nil { return err } diff --git a/pkg/network/kube_proxy.go b/pkg/network/kube_proxy.go index 31b0584e98..96efb18128 100644 --- a/pkg/network/kube_proxy.go +++ b/pkg/network/kube_proxy.go @@ -217,13 +217,9 @@ func renderStandaloneKubeProxy(conf *operv1.NetworkSpec, bootstrapResult *bootst data.Data["HealthzPort"] = healthzPort data.Data["KUBE_PROXY_NODE_SELECTOR"] = "" // DPU_DEV_PREVIEW - // Node Mode is currently configured via a stand-alone configMap and stored - // in bootstrapResult. Once out of DevPreview, CNO API will be expanded to - // include Node Mode and it will be stored in conf (operv1.NetworkSpec) and - // this code will not need to access bootstrapResult.OVN.OVNKubernetesConfig. if bootstrapResult.OVN.OVNKubernetesConfig != nil { - if bootstrapResult.OVN.OVNKubernetesConfig.NodeMode == OVN_NODE_MODE_DPU { - data.Data["KUBE_PROXY_NODE_SELECTOR"] = OVN_NODE_SELECTOR_DPU + if len(bootstrapResult.OVN.OVNKubernetesConfig.DpuModeNodes) > 0 { + data.Data["KUBE_PROXY_NODE_SELECTOR"] = bootstrapResult.OVN.OVNKubernetesConfig.DpuModeLabel + ": ''" } } diff --git a/pkg/network/kube_proxy_test.go b/pkg/network/kube_proxy_test.go index fc7d17142e..7ea40ea787 100644 --- a/pkg/network/kube_proxy_test.go +++ b/pkg/network/kube_proxy_test.go @@ -356,7 +356,10 @@ func TestFillKubeProxyDefaults(t *testing.T) { var FakeKubeProxyBootstrapResult = bootstrap.BootstrapResult{ OVN: bootstrap.OVNBootstrapResult{ OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", }, }, } diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index 62e6c50ef0..2aed4cbab1 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -59,7 +59,10 @@ const OVN_LOG_PATTERN_CONSOLE = "%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" const OVN_NODE_MODE_FULL = "full" const OVN_NODE_MODE_DPU_HOST = "dpu-host" const OVN_NODE_MODE_DPU = "dpu" -const OVN_NODE_SELECTOR_DPU = "network.operator.openshift.io/dpu: ''" +const OVN_NODE_MODE_SMART_NIC = "smart-nic" +const OVN_NODE_SELECTOR_DEFAULT_DPU_HOST = "network.operator.openshift.io/dpu-host" +const OVN_NODE_SELECTOR_DEFAULT_DPU = "network.operator.openshift.io/dpu" +const OVN_NODE_SELECTOR_DEFAULT_SMART_NIC = "network.operator.openshift.io/smart-nic" // gRPC healthcheck port. See: https://github.com/openshift/enhancements/pull/1209 const OVN_EGRESSIP_HEALTHCHECK_PORT = "9107" @@ -146,6 +149,10 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["CNIConfDir"] = pluginCNIConfDir(conf) data.Data["CNIBinDir"] = CNIBinDir data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_FULL + data.Data["DpuHostModeLabel"] = bootstrapResult.OVN.OVNKubernetesConfig.DpuHostModeLabel + data.Data["DpuModeLabel"] = bootstrapResult.OVN.OVNKubernetesConfig.DpuModeLabel + data.Data["SmartNicModeLabel"] = bootstrapResult.OVN.OVNKubernetesConfig.SmartNicModeLabel + data.Data["MgmtPortResourceName"] = bootstrapResult.OVN.OVNKubernetesConfig.MgmtPortResourceName data.Data["OVN_NB_PORT"] = OVN_NB_PORT data.Data["OVN_SB_PORT"] = OVN_SB_PORT data.Data["OVN_NB_RAFT_PORT"] = OVN_NB_RAFT_PORT @@ -382,30 +389,36 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo return nil, progressing, errors.Wrapf(err, "failed to set the status of hybrid overlay %s annotation on daemonsets or statefulsets", hybridOverlayStatus) } - nodeMode := bootstrapResult.OVN.OVNKubernetesConfig.NodeMode - if nodeMode == OVN_NODE_MODE_DPU_HOST { - data.Data["OVN_NODE_MODE"] = nodeMode + if len(bootstrapResult.OVN.OVNKubernetesConfig.SmartNicModeNodes) > 0 { + data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_SMART_NIC manifests, err = render.RenderTemplate(filepath.Join(manifestDir, manifestSubDir+"/ovnkube-node.yaml"), &data) if err != nil { - return nil, progressing, errors.Wrap(err, "failed to render manifests") + return nil, progressing, errors.Wrap(err, "failed to render manifests for smart-nic") } objs = append(objs, manifests...) - } else if nodeMode == OVN_NODE_MODE_DPU { + } + + if len(bootstrapResult.OVN.OVNKubernetesConfig.DpuHostModeNodes) > 0 { + data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_DPU_HOST + manifests, err = render.RenderTemplate(filepath.Join(manifestDir, manifestSubDir+"/ovnkube-node.yaml"), &data) + if err != nil { + return nil, progressing, errors.Wrap(err, "failed to render manifests for dpu-host") + } + objs = append(objs, manifests...) + } + + if len(bootstrapResult.OVN.OVNKubernetesConfig.DpuModeNodes) > 0 { // "OVN_NODE_MODE" not set when render.RenderDir() called above, // so render just the error-cni.yaml with "OVN_NODE_MODE" set. - data.Data["OVN_NODE_MODE"] = nodeMode + data.Data["OVN_NODE_MODE"] = OVN_NODE_MODE_DPU manifests, err = render.RenderTemplate(filepath.Join(manifestDir, "network/ovn-kubernetes/common/error-cni.yaml"), &data) if err != nil { - return nil, progressing, errors.Wrap(err, "failed to render manifests") + return nil, progressing, errors.Wrap(err, "failed to render manifests for dpu") } objs = append(objs, manifests...) // Run KubeProxy on DPU // DPU_DEV_PREVIEW - // Node Mode is currently configured via a stand-alone configMap and stored - // in bootstrapResult. Once out of DevPreview, CNO API will be expanded to - // include Node Mode and it will be stored in conf (operv1.NetworkSpec) and - // defaultDeployKubeProxy() will have access and this can be removed. if conf.DeployKubeProxy == nil { v := true conf.DeployKubeProxy = &v @@ -662,11 +675,42 @@ func getDisableUDPAggregation(cl crclient.Reader) bool { return disable } -// bootstrapOVNConfig returns the value of mode found in the openshift-ovn-kubernetes/dpu-mode-config configMap +// getNodeListByLabel returns a list of node names that matches the provided label. +func getNodeListByLabel(kubeClient cnoclient.Client, label string) ([]string, error) { + var nodeNames []string + nodeList, err := kubeClient.Default().Kubernetes().CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: label}) + if err != nil { + return nil, err + } + for _, node := range nodeList.Items { + nodeNames = append(nodeNames, node.Name) + } + klog.Infof("For Label %s, the list of nodes are %+q", label, nodeNames) + return nodeNames, nil +} + +// findCommonNode returns true if there is a common node in the node list. +func findCommonNode(nodeLists ...[]string) (bool, string) { + exists := make(map[string]bool) + for _, list := range nodeLists { + for _, value := range list { + if exists[value] { + return true, value + } + exists[value] = true + } + } + return false, "" +} + +// bootstrapOVNConfig returns the values in the openshift-ovn-kubernetes/hardware-offload-config configMap // if it exists, otherwise returns default configuration for OCP clusters using OVN-Kubernetes func bootstrapOVNConfig(conf *operv1.Network, kubeClient cnoclient.Client, hc *platform.HyperShiftConfig, infraStatus *bootstrap.InfraStatus) (*bootstrap.OVNConfigBoostrapResult, error) { ovnConfigResult := &bootstrap.OVNConfigBoostrapResult{ - NodeMode: OVN_NODE_MODE_FULL, + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", } if conf.Spec.DefaultNetwork.OVNKubernetesConfig.GatewayConfig == nil { bootstrapOVNGatewayConfig(conf, kubeClient.ClientFor("").CRClient()) @@ -679,7 +723,7 @@ func bootstrapOVNConfig(conf *operv1.Network, kubeClient cnoclient.Client, hc *p } cm := &corev1.ConfigMap{} - dmc := types.NamespacedName{Namespace: "openshift-network-operator", Name: "dpu-mode-config"} + dmc := types.NamespacedName{Namespace: "openshift-network-operator", Name: "hardware-offload-config"} err = kubeClient.ClientFor("").CRClient().Get(context.TODO(), dmc, cm) if err != nil { @@ -687,16 +731,51 @@ func bootstrapOVNConfig(conf *operv1.Network, kubeClient cnoclient.Client, hc *p return nil, fmt.Errorf("Could not determine Node Mode: %w", err) } } else { - nodeModeOverride := cm.Data["mode"] - if nodeModeOverride != OVN_NODE_MODE_DPU_HOST && nodeModeOverride != OVN_NODE_MODE_DPU { - klog.Warningf("dpu-mode-config does not match %q or %q, is: %q. Using OVN configuration: %+v", - OVN_NODE_MODE_DPU_HOST, OVN_NODE_MODE_DPU, nodeModeOverride, ovnConfigResult) - } else { - ovnConfigResult.NodeMode = nodeModeOverride - klog.Infof("Overriding OVN configuration to %+v", ovnConfigResult) + dpuHostModeLabel, exists := cm.Data["dpu-host-mode-label"] + if exists { + ovnConfigResult.DpuHostModeLabel = dpuHostModeLabel + } + + dpuModeLabel, exists := cm.Data["dpu-mode-label"] + if exists { + ovnConfigResult.DpuModeLabel = dpuModeLabel + } + + smartNicModeLabel, exists := cm.Data["smart-nic-mode-label"] + if exists { + ovnConfigResult.SmartNicModeLabel = smartNicModeLabel } + + mgmtPortresourceName, exists := cm.Data["mgmt-port-resource-name"] + if exists { + ovnConfigResult.MgmtPortResourceName = mgmtPortresourceName + } + } + + // We want to see if there are any nodes that are labeled for specific modes. + ovnConfigResult.DpuHostModeNodes, err = getNodeListByLabel(kubeClient, ovnConfigResult.DpuHostModeLabel+"=") + if err != nil { + return nil, fmt.Errorf("Could not get node list with label %s : %w", ovnConfigResult.DpuHostModeLabel, err) } + ovnConfigResult.DpuModeNodes, err = getNodeListByLabel(kubeClient, ovnConfigResult.DpuModeLabel+"=") + if err != nil { + return nil, fmt.Errorf("Could not get node list with label %s : %w", ovnConfigResult.DpuModeLabel, err) + } + + ovnConfigResult.SmartNicModeNodes, err = getNodeListByLabel(kubeClient, ovnConfigResult.SmartNicModeLabel+"=") + if err != nil { + return nil, fmt.Errorf("Could not get node list with label %s : %w", ovnConfigResult.SmartNicModeLabel, err) + } + + // No node shall have any other label set. Each node should be ONLY be DPU, DPU Host, or Smart NIC. + found, nodeName := findCommonNode(ovnConfigResult.DpuHostModeNodes, ovnConfigResult.DpuModeNodes, ovnConfigResult.SmartNicModeNodes) + if found { + return nil, fmt.Errorf("Node %s has multiple hardware offload labels.", nodeName) + } + + klog.Infof("OVN configuration is now %+v", ovnConfigResult) + ovnConfigResult.DisableUDPAggregation = getDisableUDPAggregation(kubeClient.ClientFor("").CRClient()) return ovnConfigResult, nil diff --git a/pkg/network/ovn_kubernetes_test.go b/pkg/network/ovn_kubernetes_test.go index 005b5c61c6..504e5f2069 100644 --- a/pkg/network/ovn_kubernetes_test.go +++ b/pkg/network/ovn_kubernetes_test.go @@ -75,7 +75,10 @@ func TestRenderOVNKubernetes(t *testing.T) { bootstrapResult.OVN = bootstrap.OVNBootstrapResult{ MasterAddresses: []string{"1.2.3.4", "5.6.7.8", "9.10.11.12"}, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, @@ -128,7 +131,10 @@ func TestRenderOVNKubernetesIPv6(t *testing.T) { bootstrapResult.OVN = bootstrap.OVNBootstrapResult{ MasterAddresses: []string{"1.2.3.4", "5.6.7.8", "9.10.11.12"}, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, @@ -146,7 +152,10 @@ func TestRenderOVNKubernetesIPv6(t *testing.T) { bootstrapResult.OVN = bootstrap.OVNBootstrapResult{ MasterAddresses: []string{"fd01::1", "fd01::2", "fd01::3"}, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, @@ -600,7 +609,10 @@ nodeport=true`, bootstrapResult.OVN = bootstrap.OVNBootstrapResult{ MasterAddresses: tc.masterIPs, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, @@ -1596,7 +1608,10 @@ metadata: MasterUpdateStatus: masterStatus, NodeUpdateStatus: nodeStatus, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, @@ -1899,7 +1914,10 @@ func TestRenderOVNKubernetesDualStackPrecedenceOverUpgrade(t *testing.T) { IPFamilyMode: names.IPFamilySingleStack, }, OVNKubernetesConfig: &bootstrap.OVNConfigBoostrapResult{ - NodeMode: "full", + DpuHostModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU_HOST, + DpuModeLabel: OVN_NODE_SELECTOR_DEFAULT_DPU, + SmartNicModeLabel: OVN_NODE_SELECTOR_DEFAULT_SMART_NIC, + MgmtPortResourceName: "", HyperShiftConfig: &bootstrap.OVNHyperShiftBootstrapResult{ Enabled: false, }, diff --git a/pkg/network/render.go b/pkg/network/render.go index 61e16e6e68..cb4c8b43f5 100644 --- a/pkg/network/render.go +++ b/pkg/network/render.go @@ -86,10 +86,7 @@ func Render(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.BootstrapResult // DPU_DEV_PREVIEW // There is currently a restriction that renderStandaloneKubeProxy() is // called after renderDefaultNetwork(). The OVN-Kubernetes code is enabling - // KubeProxy in Node Mode of "dpu". Once out of DevPreview, CNO API will be - // expanded to include Node Mode and it will be stored in conf (operv1.NetworkSpec) - // and KubeProxy can read Node Mode and be enabled in KubeProxy code, removing this - // dependency. + // KubeProxy in Node Mode of "dpu". o, err = renderStandaloneKubeProxy(conf, bootstrapResult, manifestDir) if err != nil { return nil, progressing, err