Supporting user-defined kubelet directory (#2893)

Co-authored-by: yuanliu <yuanliu@cmss.chinamobile.com>
kubeovn · Nov 14, 2023 · 1c94602 · 1c94602
1 parent 67adfd9
commit 1c94602
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 7 deletions.
diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -2841,7 +2841,7 @@ spec:
             - mountPath: /opt/ovs-config
               name: host-config-ovs
             - name: shareddir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: $KUBELET_DIR/pods
             - name: hugepage
               mountPath: /dev/hugepages
             - mountPath: /lib/modules
@@ -3171,6 +3171,7 @@ spec:
           - --alsologtostderr=true
           - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
           - --log_file_max_size=0
+          - --kubelet-dir=$KUBELET_DIR
         securityContext:
           runAsUser: 0
           privileged: true
@@ -3202,7 +3203,7 @@ spec:
             mountPath: /lib/modules
             readOnly: true
           - name: shared-dir
-            mountPath: /var/lib/kubelet/pods
+            mountPath: $KUBELET_DIR/pods
           - mountPath: /etc/openvswitch
             name: systemid
           - mountPath: /etc/cni/net.d

diff --git a/kubeovn-helm/templates/ovncni-ds.yaml b/kubeovn-helm/templates/ovncni-ds.yaml
@@ -72,6 +72,8 @@ spec:
           - --alsologtostderr=true
           - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
           - --log_file_max_size=0
+          - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
+          - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }}
         securityContext:
           runAsUser: 0
           privileged: true
@@ -103,7 +105,7 @@ spec:
             mountPath: /lib/modules
             readOnly: true
           - name: shared-dir
-            mountPath: /var/lib/kubelet/pods
+            mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods
           - mountPath: /etc/openvswitch
             name: systemid
           - mountPath: /etc/cni/net.d

diff --git a/kubeovn-helm/templates/ovsovn-ds.yaml b/kubeovn-helm/templates/ovsovn-ds.yaml
@@ -88,6 +88,16 @@ spec:
               name: kube-ovn-tls
             - mountPath: /var/run/containerd
               name: cruntime
+            {{- if or .Values.DPDK .Values.HYBRID_DPDK }}
+            - mountPath: /opt/ovs-config
+              name: host-config-ovs
+            - mountPath: /dev/hugepages
+              name: hugepage
+            {{- end }}
+            {{- if .Values.HYBRID_DPDK }}
+            - name: shareddir
+              mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods
+            {{- end }}
           readinessProbe:
             exec:
               command:

diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
@@ -61,6 +61,7 @@ type Configuration struct {
 	EnableVerboseConnCheck    bool
 	TCPConnCheckPort          int
 	UDPConnCheckPort          int
+	KubeletDir                string
 }
 
 // ParseFlags will parse cmd args then init kubeClient and configuration
@@ -97,6 +98,7 @@ func ParseFlags() *Configuration {
 		argEnableVerboseConnCheck   = pflag.Bool("enable-verbose-conn-check", false, "enable TCP/UDP connectivity check listen port")
 		argTCPConnectivityCheckPort = pflag.Int("tcp-conn-check-port", 8100, "TCP connectivity Check Port")
 		argUDPConnectivityCheckPort = pflag.Int("udp-conn-check-port", 8101, "UDP connectivity Check Port")
+		argKubeletDir               = pflag.String("kubelet-dir", "/var/lib/kubelet", "Path of the kubelet dir, default: /var/lib/kubelet")
 	)
 
 	// mute info log for ipset lib
@@ -149,6 +151,7 @@ func ParseFlags() *Configuration {
 		EnableVerboseConnCheck:    *argEnableVerboseConnCheck,
 		TCPConnCheckPort:          *argTCPConnectivityCheckPort,
 		UDPConnCheckPort:          *argUDPConnectivityCheckPort,
+		KubeletDir:                *argKubeletDir,
 	}
 	return config
 }

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -133,7 +133,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			nicType = util.OffloadType
 		} else if podRequest.VhostUserSocketVolumeName != "" {
 			nicType = util.DpdkType
-			if err = createShortSharedDir(pod, podRequest.VhostUserSocketVolumeName); err != nil {
+			if err = createShortSharedDir(pod, podRequest.VhostUserSocketVolumeName, csh.Config.KubeletDir); err != nil {
 				klog.Error(err.Error())
 				if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: err.Error()}); err != nil {
 					klog.Errorf("failed to write response: %v", err)

diff --git a/pkg/daemon/handler_linux.go b/pkg/daemon/handler_linux.go
@@ -22,7 +22,7 @@ func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error {
 	return nil
 }
 
-func createShortSharedDir(pod *v1.Pod, volumeName string) (err error) {
+func createShortSharedDir(pod *v1.Pod, volumeName string, kubeletDir string) (err error) {
 	var volume *v1.Volume
 	for index, v := range pod.Spec.Volumes {
 		if v.Name == volumeName {
@@ -36,7 +36,7 @@ func createShortSharedDir(pod *v1.Pod, volumeName string) (err error) {
 	if volume.EmptyDir == nil {
 		return fmt.Errorf("volume %s is not empty dir", volume.Name)
 	}
-	originSharedDir := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/%s", pod.UID, volumeName)
+	originSharedDir := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", kubeletDir, pod.UID, volumeName)
 	newSharedDir := getShortSharedDir(pod.UID, volumeName)
 	// set vhostuser dir 777 for qemu has the permission to create sock
 	mask := syscall.Umask(0)

diff --git a/pkg/daemon/handler_windows.go b/pkg/daemon/handler_windows.go
@@ -19,7 +19,7 @@ func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error {
 	return nil
 }
 
-func createShortSharedDir(pod *v1.Pod, volumeName string) error {
+func createShortSharedDir(pod *v1.Pod, volumeName string, kubeletDir string) error {
 	// nothing to do on Windows
 	return nil
 }