From 136bbb6717e84e0e9efcca02e7df512f89693b4d Mon Sep 17 00:00:00 2001 From: Michael Dent Date: Wed, 7 Jun 2023 12:34:52 +1200 Subject: [PATCH] feature/en289 Add a one-volume option for persistence using subPath Signed-off-by: Michael Dent --- README.md | 9 ++-- templates/statefulset.yaml | 106 +++++++++++++++++++++++++++++-------- values.yaml | 11 +++- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d5c51035..da4287d4 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ The following items can be set via `--set` flag during installation or configure #### Configure how to persist data -- **Disable**: The data does not survive the termination of a pod. -- **Persistent Volume Claim(default)**: A default `StorageClass` is needed in the Kubernetes cluster to dynamically provision the volumes. Specify another StorageClass in the `storageClass` or set `existingClaim` if you have already existing persistent volumes to use. +- **Disable(default)**: The data does not survive the termination of a pod. +- **Persistent Volume Claim**: Enable persistence so that data survives termination of the pod. There is the choice of using one large persistent volume (using subPath) or seven separate persistent volumes for config, data, logs, repos, etc. + A default `StorageClass` is needed in the Kubernetes cluster to dynamically provision the volumes. Specify another StorageClass in the `persistence.storageClass` setting. #### Configure authentication @@ -184,6 +185,9 @@ The following table lists the configurable parameters of the nifi chart and the | `persistence.enabled` | Use persistent volume to store data | `false` | | `persistence.storageClass` | Storage class name of PVCs (use the default type if unset) | `nil` | | `persistence.accessMode` | ReadWriteOnce or ReadOnly | `[ReadWriteOnce]` | +| `persistence.subPath.enabled` | Use only one persistent volume with subPath instead of seven separate persistent volumes | `false` | +| `persistence.subPath.name` | Name of the one persistent volume claim when using subPath | `data` | +| `persistence.subPath.size` | Size of the one persistent volume claim when using subPath | `36Gi` | | `persistence.configStorage.size` | Size of persistent volume claim | `100Mi` | | `persistence.authconfStorage.size` | Size of persistent volume claim | `100Mi` | | `persistence.dataStorage.size` | Size of persistent volume claim | `1Gi` | @@ -191,7 +195,6 @@ The following table lists the configurable parameters of the nifi chart and the | `persistence.contentRepoStorage.size` | Size of persistent volume claim | `10Gi` | | `persistence.provenanceRepoStorage.size` | Size of persistent volume claim | `10Gi` | | `persistence.logStorage.size` | Size of persistent volume claim | `5Gi` | -| `persistence.existingClaim` | Use an existing PVC to persist data | `nil` | | **jvmMemory** | | `jvmMemory` | bootstrap jvm size | `2g` | | **SideCar** | diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index 7a732024..147f5e10 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -453,20 +453,55 @@ spec: tcpSocket: port: {{ .Values.properties.httpsPort }} volumeMounts: - - name: "logs" - mountPath: /opt/nifi/nifi-current/logs - - name: "data" - mountPath: /opt/nifi/data - - name: "auth-conf" - mountPath: /opt/nifi/nifi-current/auth-conf/ - - name: "config-data" - mountPath: /opt/nifi/nifi-current/config-data - - name: "flowfile-repository" - mountPath: /opt/nifi/flowfile_repository - - name: "content-repository" - mountPath: /opt/nifi/content_repository - - name: "provenance-repository" - mountPath: /opt/nifi/provenance_repository + - mountPath: /opt/nifi/nifi-current/logs + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} + - mountPath: /opt/nifi/data + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: data + {{- else }} + name: "data" + {{- end }} + - mountPath: /opt/nifi/nifi-current/auth-conf/ + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: auth-conf + {{- else }} + name: "auth-conf" + {{- end }} + - mountPath: /opt/nifi/nifi-current/config-data + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: config-data + {{- else }} + name: "config-data" + {{- end }} + - mountPath: /opt/nifi/flowfile_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: flowfile-repository + {{- else }} + name: "flowfile-repository" + {{- end }} + - mountPath: /opt/nifi/content_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: content-repository + {{- else }} + name: "content-repository" + {{- end }} + - mountPath: /opt/nifi/provenance_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: provenance-repository + {{- else }} + name: "provenance-repository" + {{- end }} - name: "bootstrap-conf" mountPath: /opt/nifi/nifi-current/conf/bootstrap.conf subPath: "bootstrap.conf" @@ -541,8 +576,13 @@ spec: resources: {{ toYaml .Values.logresources | indent 10 }} volumeMounts: - - name: logs - mountPath: /var/log + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} - name: bootstrap-log imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" @@ -553,8 +593,13 @@ spec: resources: {{ toYaml .Values.logresources | indent 10 }} volumeMounts: - - name: logs - mountPath: /var/log + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} - name: user-log imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" @@ -565,8 +610,13 @@ spec: resources: {{ toYaml .Values.logresources | indent 10 }} volumeMounts: - - name: logs - mountPath: /var/log + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} {{- if .Values.certManager.enabled }} - name: cert-manager imagePullPolicy: {{ .Values.image.pullPolicy | quote }} @@ -786,7 +836,21 @@ spec: {{- if .Values.extraVolumes }} {{ toYaml .Values.extraVolumes | indent 6 }} {{- end }} -{{- if .Values.persistence.enabled }} +{{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ .Values.persistence.subPath.name }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.subPath.size }} +{{- end }} +{{- if and .Values.persistence.enabled (not .Values.persistence.subPath.enabled) }} volumeClaimTemplates: - metadata: name: logs diff --git a/values.yaml b/values.yaml index 158bdfed..a39199df 100644 --- a/values.yaml +++ b/values.yaml @@ -253,7 +253,16 @@ persistence: # The default storage class is used if this variable is not set. accessModes: [ReadWriteOnce] - ## Storage Capacities for persistent volumes + + ## Use subPath and have 1 persistent volume instead of 7 volumes - use when your k8s nodes have limited volume slots, to limit waste of space, + ## or your available volume sizes are quite large + # The one disk will have a directory folder for each volumeMount, but this is hidden. Run 'mount' to view each mount. + subPath: + enabled: false + name: data + size: 30Gi + + ## Storage Capacities for persistent volumes (these are ignored if using one volume with subPath) configStorage: size: 100Mi authconfStorage: