diff --git a/deploy/helm/seaweedfs-csi-driver/Chart.yaml b/deploy/helm/seaweedfs-csi-driver/Chart.yaml index 93bf795..f60c6e3 100644 --- a/deploy/helm/seaweedfs-csi-driver/Chart.yaml +++ b/deploy/helm/seaweedfs-csi-driver/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: seaweedfs-csi-driver description: A Helm chart for Kubernetes CSI backed by a SeaweedFS cluster type: application -version: 0.1.3 +version: 0.2.0 appVersion: latest diff --git a/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yaml b/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yaml index 561cb2b..854f169 100644 --- a/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yaml +++ b/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yaml @@ -3,5 +3,5 @@ kind: CSIDriver metadata: name: {{ .Values.driverName }} spec: - attachRequired: true + attachRequired: {{ .Values.csiAttacher.enabled }} podInfoOnMount: true diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yaml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yaml index d6fc537..23738d2 100644 --- a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yaml +++ b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yaml @@ -31,46 +31,21 @@ spec: tolerations: {{ toYaml . | nindent 8 }} {{- end }} containers: - - name: driver-registrar - image: {{ .Values.csiNodeDriverRegistrar.image }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - {{ toYaml .Values.csiNodeDriverRegistrar.resources | nindent 12 }} - volumeMounts: - - name: plugin-dir - mountPath: /csi/ - - name: registration-dir - mountPath: /registration/ + # SeaweedFs Plugin (node) - name: csi-seaweedfs-plugin - securityContext: - {{ toYaml .Values.seaweedfsCsiPlugin.securityContext | nindent 12 }} + securityContext: {{ toYaml .Values.seaweedfsCsiPlugin.securityContext | nindent 12 }} image: {{.Values.seaweedfsCsiPlugin.image }} imagePullPolicy: {{ .Values.imagePullPolicy }} args: - - "--endpoint=$(CSI_ENDPOINT)" - - "--filer=$(SEAWEEDFS_FILER)" - - "--nodeid=$(NODE_ID)" - - "--cacheDir=/var/cache/seaweedfs" - - "--dataLocality={{ .Values.dataLocality }}" - - "--node" - - "--controller" - - "--attacher" + - --endpoint=$(CSI_ENDPOINT) + - --filer=$(SEAWEEDFS_FILER) + - --nodeid=$(NODE_ID) + - --cacheDir=/var/cache/seaweedfs + - --dataLocality={{ .Values.dataLocality }} {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }} - - "--dataCenter=$(DATACENTER)" + - --dataCenter=$(DATACENTER) {{- end }} + - --node env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock @@ -95,12 +70,32 @@ spec: - name: WEED_GRPC_CA value: /var/run/secrets/app/tls/ca.crt {{- end }} - {{- if .Values.logVerbosity }} + {{- with .Values.logVerbosity }} - name: WEED_V - value: {{ .Values.logVerbosity | quote }} + value: {{ . | quote }} + {{- end }} + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + {{- with .Values.node.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} {{- end }} - resources: - {{ toYaml .Values.seaweedfsCsiPlugin.resources | nindent 12 }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} volumeMounts: - name: plugin-dir mountPath: /csi @@ -118,6 +113,74 @@ spec: {{- end }} - name: cache mountPath: /var/cache/seaweedfs + resources: {{ toYaml .Values.node.resources | nindent 12 }} + + # driver registrar + - name: driver-registrar + image: {{ .Values.csiNodeDriverRegistrar.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --http-endpoint=:9809 + #- --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.csiNodeDriverRegistrar.livenessProbe }} + ports: + - containerPort: 9809 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + resources: {{ toYaml .Values.csiNodeDriverRegistrar.resources | nindent 12 }} + + # liveness probe + {{- if .Values.node.livenessProbe }} + - name: csi-liveness-probe + image: {{ .Values.csiLivenessProbe.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --http-endpoint=:9808 + env: + - name: ADDRESS + value: /csi/csi.sock + ports: + - containerPort: 9808 + name: livenessprobe + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: {{ toYaml .Values.csiLivenessProbe.resources | nindent 12 }} + {{- end }} + volumes: - name: registration-dir hostPath: diff --git a/deploy/helm/seaweedfs-csi-driver/templates/deployment.yaml b/deploy/helm/seaweedfs-csi-driver/templates/deployment.yaml index 7d305ed..846a92a 100644 --- a/deploy/helm/seaweedfs-csi-driver/templates/deployment.yaml +++ b/deploy/helm/seaweedfs-csi-driver/templates/deployment.yaml @@ -7,7 +7,7 @@ spec: selector: matchLabels: app: {{ template "seaweedfs-csi-driver.name" . }}-controller - replicas: 1 + replicas: {{ .Values.controller.replicas }} template: metadata: labels: @@ -15,72 +15,39 @@ spec: spec: priorityClassName: system-cluster-critical serviceAccountName: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa - {{- with .Values.controller.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} + affinity: + {{- if and .Values.controller.affinity .Values.controller.affinity.podAffinity }} + podAffinity: {{ toYaml .Values.controller.affinity.podAffinity | nindent 10 }} + {{- end }} + {{- if and .Values.controller.affinity .Values.controller.affinity.podAntiAffinity }} + podAntiAffinity: {{ toYaml .Values.controller.affinity.podAntiAffinity | nindent 10 }} + {{- else }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ template "seaweedfs-csi-driver.name" . }}-controller + topologyKey: kubernetes.io/hostname + {{- end }} {{- with .Values.controller.tolerations }} tolerations: {{ toYaml . | nindent 8 }} {{- end }} containers: - # provisioner - - name: csi-provisioner - image: {{ .Values.csiProvisioner.image }} - args: - - "--csi-address=$(ADDRESS)" - - -v - - "9" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: {{ .Values.imagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - {{ toYaml .Values.csiProvisioner.resources | nindent 12 }} - # attacher - - name: csi-attacher - image: {{ .Values.csiAttacher.image }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--timeout=120s" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: {{ .Values.imagePullPolicy }} - resources: - {{ toYaml .Values.csiAttacher.resources | nindent 12 }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - # resizer - - name: csi-resizer - image: {{ .Values.csiResizer.image }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election=false" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: {{ .Values.imagePullPolicy }} - resources: - {{ toYaml .Values.csiAttacher.resources | nindent 12 }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - # SeaweedFs Plugin + # SeaweedFs Plugin (controller) - name: seaweedfs-csi-plugin image: {{.Values.seaweedfsCsiPlugin.image}} imagePullPolicy: {{ .Values.imagePullPolicy }} args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--filer=$(SEAWEEDFS_FILER)" - - "--nodeid=$(NODE_ID)" - - "--node" - - "--controller" - - "--attacher" + - --endpoint=$(CSI_ENDPOINT) + - --filer=$(SEAWEEDFS_FILER) + - --nodeid=$(NODE_ID)" + - --controller + {{- if .Values.csiAttacher.enabled }} + - --attacher + {{- end }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock @@ -98,10 +65,32 @@ spec: - name: WEED_GRPC_CA value: /var/run/secrets/app/tls/ca.crt {{- end }} - {{- if .Values.logVerbosity }} + {{- with .Values.logVerbosity }} - name: WEED_V - value: {{ .Values.logVerbosity | quote }} + value: {{ . | quote }} + {{- end }} + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + {{- with .Values.controller.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -109,6 +98,146 @@ spec: - name: tls mountPath: /var/run/secrets/app/tls {{- end }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + + # provisioner + - name: csi-provisioner + image: {{ .Values.csiProvisioner.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace={{ .Release.Namespace }} + - --http-endpoint=:9809 + #- --v=9 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + ports: + - containerPort: 9809 + name: healthz + {{- with .Values.csiProvisioner.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: {{ toYaml .Values.csiProvisioner.resources | nindent 12 }} + + # resizer + - name: csi-resizer + image: {{ .Values.csiResizer.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace={{ .Release.Namespace }} + - --http-endpoint=:9810 + #- --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + ports: + - containerPort: 9810 + name: healthz + {{- with .Values.csiResizer.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: {{ toYaml .Values.csiResizer.resources | nindent 12 }} + + {{- if .Values.csiAttacher.enabled }} + # attacher + - name: csi-attacher + image: {{ .Values.csiAttacher.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace={{ .Release.Namespace }} + - --http-endpoint=:9811 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + ports: + - containerPort: 9811 + name: healthz + {{- with .Values.csiAttacher.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + {{- with .failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: {{ toYaml .Values.csiAttacher.resources | nindent 12 }} + {{- end }} + + # liveness probe + {{- if .Values.controller.livenessProbe }} + - name: csi-liveness-probe + image: {{ .Values.csiLivenessProbe.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --http-endpoint=:9808 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + ports: + - containerPort: 9808 + name: livenessprobe + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: {{ toYaml .Values.csiLivenessProbe.resources | nindent 12 }} + {{- end }} + volumes: - name: socket-dir emptyDir: {} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/rbac.yaml b/deploy/helm/seaweedfs-csi-driver/templates/rbac.yaml index f373717..0fe36cb 100644 --- a/deploy/helm/seaweedfs-csi-driver/templates/rbac.yaml +++ b/deploy/helm/seaweedfs-csi-driver/templates/rbac.yaml @@ -31,7 +31,6 @@ rules: - apiGroups: [ "" ] resources: [ "pods" ] verbs: [ "get", "list", "watch" ] - --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -45,6 +44,8 @@ roleRef: kind: ClusterRole name: {{ template "seaweedfs-csi-driver.name" . }}-provisioner-role apiGroup: rbac.authorization.k8s.io + +{{- if .Values.csiAttacher.enabled }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -63,7 +64,6 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments", "volumeattachments/status"] verbs: ["get", "list", "watch", "update", "patch"] - --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -77,6 +77,9 @@ roleRef: kind: ClusterRole name: {{ template "seaweedfs-csi-driver.name" . }}-attacher-role apiGroup: rbac.authorization.k8s.io +{{- end }} + +{{- if .Values.csiSnapshotter.enabled }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -123,6 +126,8 @@ roleRef: kind: ClusterRole name: {{ template "seaweedfs-csi-driver.name" . }}-snapshotter-role apiGroup: rbac.authorization.k8s.io +{{- end }} + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -148,6 +153,7 @@ roleRef: kind: ClusterRole name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-controller-role apiGroup: rbac.authorization.k8s.io + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -176,3 +182,28 @@ roleRef: kind: ClusterRole name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-node-role apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-binding + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa +roleRef: + kind: Role + name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/helm/seaweedfs-csi-driver/values.yaml b/deploy/helm/seaweedfs-csi-driver/values.yaml index bb29782..ac25b0c 100644 --- a/deploy/helm/seaweedfs-csi-driver/values.yaml +++ b/deploy/helm/seaweedfs-csi-driver/values.yaml @@ -11,24 +11,56 @@ imagePullPolicy: "IfNotPresent" #- name: mycredentials csiProvisioner: - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 resources: {} + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 -csiAttacher: - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 +csiResizer: + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 resources: {} + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 -csiResizer: - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 +csiAttacher: + # generally we don't need attacher, cause we do nothing to attach volume to node + # we will keep this for a historical reason + # you need to delete seaweedfs CSIDriver crd manually before upgrading with attacher disabled + # also you need to delete all seaweedfs VolumeAttachment crd manually after upgrade + enabled: true + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 resources: {} + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 + +csiSnapshotter: + # we do not support snapshots yet + enabled: false csiNodeDriverRegistrar: - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + resources: {} + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 + +csiLivenessProbe: + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 resources: {} seaweedfsCsiPlugin: image: chrislusf/seaweedfs-csi-driver:latest - resources: {} securityContext: privileged: true capabilities: @@ -39,8 +71,15 @@ seaweedfsCsiPlugin: driverName: seaweedfs-csi-driver controller: + replicas: 1 affinity: {} - tolerations: {} + tolerations: + resources: {} + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 # DataLocality (inspired by Longhorn) allows instructing the storage-driver which volume-locations will be used or preferred in Pods to read & write. # e.g. Allows Pods to write preferrably to its local dataCenter volume-servers @@ -58,7 +97,12 @@ node: rollingUpdate: maxUnavailable: 25% affinity: {} - tolerations: {} + tolerations: + livenessProbe: + failureThreshold: + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 # Auto-Inject Topology-Info from Kubernetes node-labels using KubeMod (https://github.com/kubemod/kubemod) # Necessary because DownwardAPI doesnt support passing node-labels (see: https://github.com/kubernetes/kubernetes/issues/40610) @@ -73,3 +117,4 @@ node: registration_dir: /var/lib/kubelet/plugins_registry plugins_dir: /var/lib/kubelet/plugins pods_mount_dir: /var/lib/kubelet/pods + diff --git a/deploy/kubernetes/seaweedfs-csi.yaml b/deploy/kubernetes/seaweedfs-csi.yaml index 276dcfd..6285419 100644 --- a/deploy/kubernetes/seaweedfs-csi.yaml +++ b/deploy/kubernetes/seaweedfs-csi.yaml @@ -1,17 +1,17 @@ --- -# Source: seaweedfs-csi-driver/templates/serviceaccounts.yml +# Source: seaweedfs-csi-driver/templates/serviceaccounts.yaml apiVersion: v1 kind: ServiceAccount metadata: name: seaweedfs-controller-sa --- -# Source: seaweedfs-csi-driver/templates/serviceaccounts.yml +# Source: seaweedfs-csi-driver/templates/serviceaccounts.yaml apiVersion: v1 kind: ServiceAccount metadata: name: seaweedfs-node-sa --- -# Source: seaweedfs-csi-driver/templates/storageclass.yml +# Source: seaweedfs-csi-driver/templates/storageclass.yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: @@ -20,7 +20,7 @@ metadata: provisioner: seaweedfs-csi-driver allowVolumeExpansion: true --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -54,7 +54,7 @@ rules: resources: [ "pods" ] verbs: [ "get", "list", "watch" ] --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -73,41 +73,7 @@ rules: resources: ["volumeattachments", "volumeattachments/status"] verbs: ["get", "list", "watch", "update", "patch"] --- -# Source: seaweedfs-csi-driver/templates/rbac.yml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: seaweedfs-snapshotter-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] ---- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -120,7 +86,7 @@ rules: resources: ["csinodes"] verbs: ["get", "list", "watch"] --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -136,7 +102,7 @@ rules: resources: ["nodes"] verbs: ["get", "list", "watch"] --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -144,12 +110,13 @@ metadata: subjects: - kind: ServiceAccount name: seaweedfs-controller-sa + namespace: default roleRef: kind: ClusterRole name: seaweedfs-provisioner-role apiGroup: rbac.authorization.k8s.io --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -157,51 +124,67 @@ metadata: subjects: - kind: ServiceAccount name: seaweedfs-controller-sa + namespace: default roleRef: kind: ClusterRole name: seaweedfs-attacher-role apiGroup: rbac.authorization.k8s.io --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: seaweedfs-snapshotter-binding + name: seaweedfs-driver-registrar-controller-binding subjects: - kind: ServiceAccount name: seaweedfs-controller-sa + namespace: default roleRef: kind: ClusterRole - name: seaweedfs-snapshotter-role + name: seaweedfs-driver-registrar-controller-role apiGroup: rbac.authorization.k8s.io --- -# Source: seaweedfs-csi-driver/templates/rbac.yml +# Source: seaweedfs-csi-driver/templates/rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: seaweedfs-driver-registrar-controller-binding + name: seaweedfs-driver-registrar-node-binding subjects: - kind: ServiceAccount - name: seaweedfs-controller-sa + name: seaweedfs-node-sa + namespace: default roleRef: kind: ClusterRole - name: seaweedfs-driver-registrar-controller-role + name: seaweedfs-driver-registrar-node-role apiGroup: rbac.authorization.k8s.io --- -# Source: seaweedfs-csi-driver/templates/rbac.yml -kind: ClusterRoleBinding +# Source: seaweedfs-csi-driver/templates/rbac.yaml +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: seaweedfs-driver-registrar-node-binding + name: seaweedfs-leader-election-controller-role + namespace: default +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +# Source: seaweedfs-csi-driver/templates/rbac.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: seaweedfs-leader-election-controller-binding + namespace: default subjects: - kind: ServiceAccount - name: seaweedfs-node-sa + namespace: default + name: seaweedfs-controller-sa roleRef: - kind: ClusterRole - name: seaweedfs-driver-registrar-node-role + kind: Role + name: seaweedfs-leader-election-controller-role apiGroup: rbac.authorization.k8s.io --- -# Source: seaweedfs-csi-driver/templates/daemonset.yml +# Source: seaweedfs-csi-driver/templates/daemonset.yaml kind: DaemonSet apiVersion: apps/v1 metadata: @@ -225,13 +208,68 @@ spec: #hostNetwork: true #dnsPolicy: ClusterFirstWithHostNet containers: + # SeaweedFs Plugin (node) + - name: csi-seaweedfs-plugin + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + image: chrislusf/seaweedfs-csi-driver:latest + imagePullPolicy: IfNotPresent + args: + - --endpoint=$(CSI_ENDPOINT) + - --filer=$(SEAWEEDFS_FILER) + - --nodeid=$(NODE_ID) + - --cacheDir=/var/cache/seaweedfs + - --dataLocality=none + - --node + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: SEAWEEDFS_FILER + value: "SEAWEEDFS_FILER:8888" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: plugins-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /dev + name: device-dir + - name: cache + mountPath: /var/cache/seaweedfs + resources: + null + + # driver registrar - name: driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 imagePullPolicy: IfNotPresent args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --http-endpoint=:9809 + #- --v=5 env: - name: ADDRESS value: /csi/csi.sock @@ -241,55 +279,56 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - resources: - - {} + ports: + - containerPort: 9809 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 volumeMounts: - name: plugin-dir mountPath: /csi/ - name: registration-dir mountPath: /registration/ - - name: csi-seaweedfs-plugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: chrislusf/seaweedfs-csi-driver:latest + resources: + {} + + # liveness probe + - name: csi-liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 imagePullPolicy: IfNotPresent - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--filer=$(SEAWEEDFS_FILER)" - - "--nodeid=$(NODE_ID)" + args: + - --csi-address=$(ADDRESS) + - --http-endpoint=:9808 env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: SEAWEEDFS_FILER - value: "SEAWEEDFS_FILER:8888" - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - - {} + - name: ADDRESS + value: /csi/csi.sock + ports: + - containerPort: 9808 + name: livenessprobe volumeMounts: - name: plugin-dir mountPath: /csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: "Bidirectional" - - mountPath: /dev - name: device-dir + resources: + {} + volumes: - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: /var/lib/kubelet/plugins_registry type: DirectoryOrCreate - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/seaweedfs-csi-driver type: DirectoryOrCreate + - name: plugins-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory - name: pods-mount-dir hostPath: path: /var/lib/kubelet/pods @@ -297,9 +336,11 @@ spec: - name: device-dir hostPath: path: /dev + - name: cache + emptyDir: {} --- -# Source: seaweedfs-csi-driver/templates/statefulset.yml -kind: StatefulSet +# Source: seaweedfs-csi-driver/templates/deployment.yaml +kind: Deployment apiVersion: apps/v1 metadata: name: seaweedfs-controller @@ -307,7 +348,6 @@ spec: selector: matchLabels: app: seaweedfs-controller - serviceName: "csi-seaweedfs" replicas: 1 template: metadata: @@ -316,83 +356,165 @@ spec: spec: priorityClassName: system-cluster-critical serviceAccountName: seaweedfs-controller-sa + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - seaweedfs-controller + topologyKey: kubernetes.io/hostname containers: + # SeaweedFs Plugin (controller) + - name: seaweedfs-csi-plugin + image: chrislusf/seaweedfs-csi-driver:latest + imagePullPolicy: IfNotPresent + args : + - --endpoint=$(CSI_ENDPOINT) + - --filer=$(SEAWEEDFS_FILER) + - --nodeid=$(NODE_ID)" + - --controller + - --attacher + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: SEAWEEDFS_FILER + value: "SEAWEEDFS_FILER:8888" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {} + # provisioner - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + imagePullPolicy: IfNotPresent args: - - "--csi-address=$(ADDRESS)" - - -v - - "9" + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace=default + - --http-endpoint=:9809 + #- --v=9 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: IfNotPresent + ports: + - containerPort: 9809 + name: healthz + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - + resources: {} - # attacher - - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + + # resizer + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + imagePullPolicy: IfNotPresent args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--timeout=120s" + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace=default + - --http-endpoint=:9810 + #- --v=5 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: IfNotPresent - resources: - - {} + ports: + - containerPort: 9810 + name: healthz + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - # resizer - - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + resources: + {} + # attacher + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + imagePullPolicy: IfNotPresent args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election=false" + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace=default + - --http-endpoint=:9811 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: IfNotPresent - resources: - - {} + ports: + - containerPort: 9811 + name: healthz + livenessProbe: + httpGet: + path: /healthz/leader-election + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 60 volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - # SeaweedFs Plugin - - name: seaweedfs-csi-plugin - image: chrislusf/seaweedfs-csi-driver:latest + resources: + {} + + # liveness probe + - name: csi-liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 imagePullPolicy: IfNotPresent - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--filer=$(SEAWEEDFS_FILER)" - - "--nodeid=$(NODE_ID)" + args: + - --csi-address=$(ADDRESS) + - --http-endpoint=:9808 env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: SEAWEEDFS_FILER - value: "SEAWEEDFS_FILER:8888" - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + ports: + - containerPort: 9808 + name: livenessprobe volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {} + volumes: - name: socket-dir emptyDir: {} --- -# Source: seaweedfs-csi-driver/templates/csidriver.yml +# Source: seaweedfs-csi-driver/templates/kubemod_modrule.yaml +# Based on https://github.com/kubernetes/kubernetes/issues/40610#issuecomment-1364368282 +--- +# Source: seaweedfs-csi-driver/templates/csidriver.yaml apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: