Skip to content

OCPBUGS-12856: Support Device Plugin Resources For Smart NIC and DPU Hosts #1721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 1 commit into from
May 6, 2023
Merged

OCPBUGS-12856: Support Device Plugin Resources For Smart NIC and DPU Hosts #1721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 41 additions & 5 deletions bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ apiVersion: apps/v1
metadata:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
name: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
name: ovnkube-node-smart-nic
{{ else }}
name: ovnkube-node
{{ end }}
Expand All @@ -17,6 +19,8 @@ spec:
matchLabels:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
app: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
app: ovnkube-node-smart-nic
{{ else }}
app: ovnkube-node
{{ end }}
Expand All @@ -31,6 +35,8 @@ spec:
labels:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
app: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
app: ovnkube-node-smart-nic
{{ else }}
app: ovnkube-node
{{ end }}
Expand All @@ -44,14 +50,30 @@ spec:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: network.operator.openshift.io/dpu-host
{{ if .DpuHostModeLabel }}
- key: {{ .DpuHostModeLabel }}
{{ if eq .OVN_NODE_MODE "dpu-host" }}
operator: Exists
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
operator: DoesNotExist
{{ else }}
operator: DoesNotExist
{{ end }}
- key: network.operator.openshift.io/dpu
{{ end }}
{{ if .SmartNicModeLabel }}
- key: {{ .SmartNicModeLabel }}
{{ if eq .OVN_NODE_MODE "dpu-host" }}
operator: DoesNotExist
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
operator: Exists
{{ else }}
operator: DoesNotExist
{{ end }}
{{ end }}
{{ if .DpuModeLabel }}
- key: {{ .DpuModeLabel }}
operator: DoesNotExist
{{ end }}
serviceAccountName: ovn-kubernetes-node
hostNetwork: true
dnsPolicy: Default
Expand All @@ -69,7 +91,7 @@ spec:
{{end}}
initContainers:
# ovnkube-node-init: wait for sbdb ready
{{ if eq .OVN_NODE_MODE "full" }}
{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }}
- name: ovnkube-node-init
image: "{{.OvnImage}}"
command:
Expand Down Expand Up @@ -122,7 +144,7 @@ spec:
# /run/openvswitch -> tmpfs - ovsdb sockets
# /env -> configmap env-overrides - debug overrides
containers:
{{ if eq .OVN_NODE_MODE "full" }}
{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }}
{{if .ENABLE_OVN_NODE_PROXY}}
# ovnkube-node-proxy redirects ovn sbdb traffic to http proxy
- name: ovnkube-node-proxy
Expand Down Expand Up @@ -430,6 +452,9 @@ spec:
if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then
node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}"
fi
if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then
node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}"
fi

multi_network_enabled_flag=
if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then
Expand Down Expand Up @@ -509,6 +534,10 @@ spec:
- name: IPFIX_SAMPLING
value: "{{.IPFIXSampling}}"
{{ end }}
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
- name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME
value: {{ .MgmtPortResourceName }}
{{ end }}
- name: K8S_NODE
valueFrom:
fieldRef:
Expand Down Expand Up @@ -573,6 +602,13 @@ spec:
requests:
cpu: 10m
memory: 300Mi
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
{{ .MgmtPortResourceName }}: '1'
{{ end }}
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
limits:
{{ .MgmtPortResourceName }}: '1'
{{ end }}
lifecycle:
preStop:
exec:
Expand Down Expand Up @@ -676,7 +712,7 @@ spec:
- name: run-ovn
hostPath:
path: /var/run/ovn
{{ if eq .OVN_NODE_MODE "full" }}
{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }}
# Used for placement of ACL audit logs
- name: node-log
hostPath:
Expand Down
74 changes: 55 additions & 19 deletions bindata/network/ovn-kubernetes/self-hosted/ovnkube-node.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ apiVersion: apps/v1
metadata:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
name: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
name: ovnkube-node-smart-nic
{{ else }}
name: ovnkube-node
{{ end }}
Expand All @@ -17,6 +19,8 @@ spec:
matchLabels:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
app: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
app: ovnkube-node-smart-nic
{{ else }}
app: ovnkube-node
{{ end }}
Expand All @@ -31,6 +35,8 @@ spec:
labels:
{{ if eq .OVN_NODE_MODE "dpu-host" }}
app: ovnkube-node-dpu-host
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
app: ovnkube-node-smart-nic
{{ else }}
app: ovnkube-node
{{ end }}
Expand All @@ -44,14 +50,30 @@ spec:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: network.operator.openshift.io/dpu-host
{{ if .DpuHostModeLabel }}
- key: {{ .DpuHostModeLabel }}
{{ if eq .OVN_NODE_MODE "dpu-host" }}
operator: Exists
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
operator: DoesNotExist
{{ else }}
operator: DoesNotExist
{{ end }}
- key: network.operator.openshift.io/dpu
{{ end }}
{{ if .SmartNicModeLabel }}
- key: {{ .SmartNicModeLabel }}
{{ if eq .OVN_NODE_MODE "dpu-host" }}
operator: DoesNotExist
{{ else if eq .OVN_NODE_MODE "smart-nic" }}
operator: Exists
{{ else }}
operator: DoesNotExist
{{ end }}
{{ end }}
{{ if .DpuModeLabel }}
- key: {{ .DpuModeLabel }}
operator: DoesNotExist
{{ end }}
serviceAccountName: ovn-kubernetes-node
hostNetwork: true
dnsPolicy: Default
Expand All @@ -64,7 +86,7 @@ spec:
# /run/openvswitch -> tmpfs - ovsdb sockets
# /env -> configmap env-overrides - debug overrides
containers:
{{ if eq .OVN_NODE_MODE "full" }}
{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }}
# ovn-controller: programs the vswitch with flows from the sbdb
- name: ovn-controller
image: "{{.OvnImage}}"
Expand All @@ -77,8 +99,8 @@ spec:
set -o allexport
source "/env/${K8S_NODE}"
set +o allexport
fi
fi

echo "$(date -Iseconds) - starting ovn-controller"
exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
--no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \
Expand Down Expand Up @@ -134,7 +156,7 @@ spec:
set -euo pipefail

# Rotate audit log files when then get to max size (in bytes)
MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 ))
MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 ))
LOGFILE=/var/log/ovn/acl-audit-log.log
CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)

Expand All @@ -143,14 +165,14 @@ spec:

while true
do
# Make sure ovn-controller's logfile exists, and get current size in bytes
if [ -f "$LOGFILE" ]; then
# Make sure ovn-controller's logfile exists, and get current size in bytes
if [ -f "$LOGFILE" ]; then
file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
else
else
ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen
file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
fi
fi

if [ $file_size -gt $MAXFILESIZE ];then
echo "Rotating OVN ACL Log File"
timestamp=`date '+%Y-%m-%dT%H-%M-%S'`
Expand All @@ -159,8 +181,8 @@ spec:
CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)
fi

# sleep for 30 seconds to avoid wasting CPU
sleep 30
# sleep for 30 seconds to avoid wasting CPU
sleep 30
done
resources:
requests:
Expand Down Expand Up @@ -190,7 +212,7 @@ spec:
TS=$(date +%s)
WARN_TS=$(( ${TS} + $(( 20 * 60)) ))
HAS_LOGGED_INFO=0

log_missing_certs(){
CUR_TS=$(date +%s)
if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then
Expand All @@ -204,7 +226,7 @@ spec:
log_missing_certs
sleep 5
done

echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy
exec /usr/bin/kube-rbac-proxy \
--logtostderr \
Expand Down Expand Up @@ -337,6 +359,9 @@ spec:
if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then
node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}"
fi
if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then
node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}"
fi

multi_network_enabled_flag=
if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then
Expand Down Expand Up @@ -404,6 +429,10 @@ spec:
- name: IPFIX_SAMPLING
value: "{{.IPFIXSampling}}"
{{ end }}
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
- name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME
value: {{ .MgmtPortResourceName }}
{{ end }}
- name: K8S_NODE
valueFrom:
fieldRef:
Expand Down Expand Up @@ -468,6 +497,13 @@ spec:
requests:
cpu: 10m
memory: 300Mi
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
{{ .MgmtPortResourceName }}: '1'
{{ end }}
{{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }}
limits:
{{ .MgmtPortResourceName }}: '1'
{{ end }}
lifecycle:
preStop:
exec:
Expand Down Expand Up @@ -571,13 +607,13 @@ spec:
- name: run-ovn
hostPath:
path: /var/run/ovn
{{ if eq .OVN_NODE_MODE "full" }}
# Used for placement of ACL audit logs
{{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }}
# Used for placement of ACL audit logs
- name: node-log
hostPath:
hostPath:
path: /var/log/ovn
- name: log-socket
hostPath:
hostPath:
path: /dev/log
{{ end }}
# For CNI server
Expand Down
9 changes: 0 additions & 9 deletions hack/dpu-mode.yaml
View file
Open in desktop

This file was deleted.

11 changes: 11 additions & 0 deletions hack/hardware-offload-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Example ConfigMap to watch for specific label names and resource names for hardware offloading with OVNKubernetes
apiVersion: v1
kind: ConfigMap
metadata:
name: hardware-offload-config
namespace: openshift-network-operator
data:
dpu-host-mode-label: "network.operator.openshift.io/dpu-host"
dpu-mode-label: "network.operator.openshift.io/dpu"
smart-nic-mode-label: "network.operator.openshift.io/smart-nic"
mgmt-port-resource-name: "openshift.io/mgmtvf"
8 changes: 7 additions & 1 deletion pkg/bootstrap/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,15 @@ type OVNHyperShiftBootstrapResult struct {

type OVNConfigBoostrapResult struct {
GatewayMode string
NodeMode string
HyperShiftConfig *OVNHyperShiftBootstrapResult
DisableUDPAggregation bool
DpuHostModeLabel string
DpuHostModeNodes []string
DpuModeLabel string
DpuModeNodes []string
SmartNicModeLabel string
SmartNicModeNodes []string
MgmtPortResourceName string
}

// OVNUpdateStatus contains the status of existing daemonset
Expand Down
10 changes: 6 additions & 4 deletions pkg/controller/operconfig/operconfig_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,15 @@ func add(mgr manager.Manager, r *ReconcileOperConfig) error {
return err
}

// Watch when nodes are created too
newNodePredicate := predicate.Funcs{
// Watch when nodes are created and updated.
// We need to watch when nodes are updated since we are interested in the labels
// of nodes for hardware offloading.
nodePredicate := predicate.Funcs{
CreateFunc: func(_ event.CreateEvent) bool {
return true
},
UpdateFunc: func(_ event.UpdateEvent) bool {
return false
return true
},
DeleteFunc: func(_ event.DeleteEvent) bool {
return true
Expand All @@ -132,7 +134,7 @@ func add(mgr manager.Manager, r *ReconcileOperConfig) error {
if err := c.Watch(
&source.Kind{Type: &corev1.Node{}},
handler.EnqueueRequestsFromMapFunc(reconcileOperConfig),
newNodePredicate,
nodePredicate,
); err != nil {
return err
}
Expand Down
8 changes: 2 additions & 6 deletions pkg/network/kube_proxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,9 @@ func renderStandaloneKubeProxy(conf *operv1.NetworkSpec, bootstrapResult *bootst
data.Data["HealthzPort"] = healthzPort
data.Data["KUBE_PROXY_NODE_SELECTOR"] = ""
// DPU_DEV_PREVIEW
// Node Mode is currently configured via a stand-alone configMap and stored
// in bootstrapResult. Once out of DevPreview, CNO API will be expanded to
// include Node Mode and it will be stored in conf (operv1.NetworkSpec) and
// this code will not need to access bootstrapResult.OVN.OVNKubernetesConfig.
if bootstrapResult.OVN.OVNKubernetesConfig != nil {
if bootstrapResult.OVN.OVNKubernetesConfig.NodeMode == OVN_NODE_MODE_DPU {
data.Data["KUBE_PROXY_NODE_SELECTOR"] = OVN_NODE_SELECTOR_DPU
if len(bootstrapResult.OVN.OVNKubernetesConfig.DpuModeNodes) > 0 {
data.Data["KUBE_PROXY_NODE_SELECTOR"] = bootstrapResult.OVN.OVNKubernetesConfig.DpuModeLabel + ": ''"
}
}

Expand Down
Loading