Create Job template for DataChain workers

charts/studio/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: studio
 description: A Helm chart for Kubernetes
 type: application
-version: 0.16.58
+version: 0.16.59
 appVersion: "v2.148.3"
 maintainers:
   - name: iterative

charts/studio/README.md

@@ -1,6 +1,6 @@
 # studio
 
-![Version: 0.16.58](https://img.shields.io/badge/Version-0.16.58-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.148.3](https://img.shields.io/badge/AppVersion-v2.148.3-informational?style=flat-square)
+![Version: 0.16.59](https://img.shields.io/badge/Version-0.16.59-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.148.3](https://img.shields.io/badge/AppVersion-v2.148.3-informational?style=flat-square)
 
 A Helm chart for Kubernetes
 
@@ -139,7 +139,7 @@ A Helm chart for Kubernetes
 | studioBlobvault.image | object | `{"repository":"nginx","tag":"1.27.0-alpine"}` | Image to use for the blobvault service |
 | studioBlobvault.image.repository | string | `"nginx"` | Image repository |
 | studioBlobvault.image.tag | string | `"1.27.0-alpine"` | Image tag |
-| studioDatachainWorker | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":5,"minReplicas":1,"targetCPUUtilizationPercentage":80,"template":[]},"envFromSecret":"","envVars":{},"ephemeralStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker","storageClass":""},"size":"20Gi","type":"ephemeral"},"image":{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-datachain-worker"},"jobQuota":10,"localStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker-local","storageClass":""},"size":"50Gi","type":"ephemeral"},"logLevel":"info","nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"ephemeral-storage":"20Gi","memory":"16Gi"},"requests":{"cpu":"1000m","ephemeral-storage":"10Gi","memory":"3Gi"}},"securityContext":{},"serviceAccount":{"annotations":{},"create":false,"name":""},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":0}},"terminationGracePeriodSeconds":180,"tolerations":[]}` | Studio DataChain Worker settings group |
+| studioDatachainWorker | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":5,"minReplicas":1,"targetCPUUtilizationPercentage":80,"template":[]},"envFromSecret":"","envVars":{},"ephemeralStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker","storageClass":""},"size":"20Gi","type":"ephemeral"},"image":{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-datachain-worker"},"localStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker-local","storageClass":""},"size":"50Gi","type":"ephemeral"},"logLevel":"info","nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"ephemeral-storage":"20Gi","memory":"16Gi"},"requests":{"cpu":"1000m","ephemeral-storage":"10Gi","memory":"3Gi"}},"securityContext":{},"serviceAccount":{"annotations":{},"create":false,"name":""},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":0}},"terminationGracePeriodSeconds":180,"tolerations":[]}` | Studio DataChain Worker settings group |
 | studioDatachainWorker.affinity | object | `{}` | DataChain worker pod affinity configuration |
 | studioDatachainWorker.autoscaling | object | `{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":5,"minReplicas":1,"targetCPUUtilizationPercentage":80,"template":[]}` | DataChain worker autoscaling configuration |
 | studioDatachainWorker.autoscaling.annotations | object | `{}` | Worker autoscaling annotation |
@@ -160,7 +160,6 @@ A Helm chart for Kubernetes
 | studioDatachainWorker.image | object | `{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-datachain-worker"}` | DataChain worker image settings |
 | studioDatachainWorker.image.pullPolicy | string | `"IfNotPresent"` | DataChain worker image pull policy |
 | studioDatachainWorker.image.repository | string | `"docker.iterative.ai/studio-datachain-worker"` | DataChain worker image repository |
-| studioDatachainWorker.jobQuota | int | `10` | DataChain worker job quota |
 | studioDatachainWorker.localStorage | object | `{"persistentVolumeClaim":{"claimName":"datachain-worker-local","storageClass":""},"size":"50Gi","type":"ephemeral"}` | Local storage configuration (used for storing DataChain virtual environments) |
 | studioDatachainWorker.localStorage.persistentVolumeClaim | object | `{"claimName":"datachain-worker-local","storageClass":""}` | Persistent Volume Claim configuration for local storage |
 | studioDatachainWorker.localStorage.persistentVolumeClaim.claimName | string | `"datachain-worker-local"` | Persistent Volume Claim name, to mount externally managed PVC (`localStorage.type` has to be set to `pvc`) |
@@ -178,6 +177,38 @@ A Helm chart for Kubernetes
 | studioDatachainWorker.strategy | object | `{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":0}}` | DataChain worker deployment strategy |
 | studioDatachainWorker.terminationGracePeriodSeconds | int | `180` | DataChain worker termination grace period |
 | studioDatachainWorker.tolerations | list | `[]` | DataChain worker pod tolerations configuration |
+| studioDatachainWorkerJobTemplate | object | `{"activeDeadlineSeconds":86400,"affinity":{},"backoffLimit":0,"envVars":{},"ephemeralStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker","storageClass":""},"size":"20Gi","type":"ephemeral"},"image":{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-datachain-worker"},"jobQuota":10,"localStorage":{"persistentVolumeClaim":{"claimName":"datachain-worker-local","storageClass":""},"size":"50Gi","type":"ephemeral"},"logLevel":"info","nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"resources":{"limits":{"ephemeral-storage":"20Gi","memory":"16Gi"},"requests":{"cpu":"1000m","ephemeral-storage":"10Gi","memory":"3Gi"}},"runtimeClassName":null,"securityContext":{},"serviceAccount":{"annotations":{},"create":false,"name":""},"tolerations":[],"ttlSecondsAfterFinished":5}` | Studio DataChain Worker Job template settings group |
+| studioDatachainWorkerJobTemplate.activeDeadlineSeconds | int | `86400` | DataChain worker Job active deadline (seconds) |
+| studioDatachainWorkerJobTemplate.affinity | object | `{}` | DataChain worker Job pod affinity configuration |
+| studioDatachainWorkerJobTemplate.backoffLimit | int | `0` | DataChain worker Job backoff limit |
+| studioDatachainWorkerJobTemplate.envVars | object | `{}` | Additional environment variables for DataChain worker Job pods |
+| studioDatachainWorkerJobTemplate.ephemeralStorage | object | `{"persistentVolumeClaim":{"claimName":"datachain-worker","storageClass":""},"size":"20Gi","type":"ephemeral"}` | Ephemeral storage configuration |
+| studioDatachainWorkerJobTemplate.ephemeralStorage.persistentVolumeClaim | object | `{"claimName":"datachain-worker","storageClass":""}` | Persistent Volume Claim configuration for ephemeral storage (if type is set `pvc`) |
+| studioDatachainWorkerJobTemplate.ephemeralStorage.persistentVolumeClaim.claimName | string | `"datachain-worker"` | Persistent Volume Claim name, to mount externally managed PVC (`ephemeralStorage.type` has to be set to `pvc`) |
+| studioDatachainWorkerJobTemplate.ephemeralStorage.persistentVolumeClaim.storageClass | string | `""` | Persistent Volume Claim `storageClass` name, by default it will use the default `storageClass`(not used when `pvc` is set as type) |
+| studioDatachainWorkerJobTemplate.ephemeralStorage.size | string | `"20Gi"` | Ephemeral Storage size |
+| studioDatachainWorkerJobTemplate.ephemeralStorage.type | string | `"ephemeral"` | Ephemeral Storage type. Possible values: `emptyDir`,  `ephemeral`, `pvc`, `pvcRWX` |
+| studioDatachainWorkerJobTemplate.image | object | `{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-datachain-worker"}` | DataChain worker Job image settings |
+| studioDatachainWorkerJobTemplate.image.pullPolicy | string | `"IfNotPresent"` | DataChain worker Job image pull policy |
+| studioDatachainWorkerJobTemplate.image.repository | string | `"docker.iterative.ai/studio-datachain-worker"` | DataChain worker Job image repository |
+| studioDatachainWorkerJobTemplate.jobQuota | int | `10` | DataChain worker Job quota |
+| studioDatachainWorkerJobTemplate.localStorage | object | `{"persistentVolumeClaim":{"claimName":"datachain-worker-local","storageClass":""},"size":"50Gi","type":"ephemeral"}` | Local storage configuration (used for storing DataChain virtual environments) |
+| studioDatachainWorkerJobTemplate.localStorage.persistentVolumeClaim | object | `{"claimName":"datachain-worker-local","storageClass":""}` | Persistent Volume Claim configuration for local storage |
+| studioDatachainWorkerJobTemplate.localStorage.persistentVolumeClaim.claimName | string | `"datachain-worker-local"` | Persistent Volume Claim name, to mount externally managed PVC (`localStorage.type` has to be set to `pvc`) |
+| studioDatachainWorkerJobTemplate.localStorage.persistentVolumeClaim.storageClass | string | `""` | Persistent Volume Claim `storageClass` name, by default it will use the default `storageClass`(not used when `pvc` is set as type) |
+| studioDatachainWorkerJobTemplate.localStorage.size | string | `"50Gi"` | Local Storage size |
+| studioDatachainWorkerJobTemplate.localStorage.type | string | `"ephemeral"` | Local Storage type. Possible values: `emptyDir`,  `ephemeral`, `pvc` |
+| studioDatachainWorkerJobTemplate.logLevel | string | `"info"` | DataChain worker Job log level |
+| studioDatachainWorkerJobTemplate.nodeSelector | object | `{}` | DataChain worker Job pod node selector configuration |
+| studioDatachainWorkerJobTemplate.podAnnotations | object | `{}` | Additional DataChain worker Job pod annotations |
+| studioDatachainWorkerJobTemplate.podSecurityContext | object | `{}` | DataChain worker Job pod security context configuration |
+| studioDatachainWorkerJobTemplate.resources | object | `{"limits":{"ephemeral-storage":"20Gi","memory":"16Gi"},"requests":{"cpu":"1000m","ephemeral-storage":"10Gi","memory":"3Gi"}}` | DataChain worker Job resources configuration |
+| studioDatachainWorkerJobTemplate.resources.limits | object | `{"ephemeral-storage":"20Gi","memory":"16Gi"}` | DataChain worker Job limits configuration |
+| studioDatachainWorkerJobTemplate.resources.requests | object | `{"cpu":"1000m","ephemeral-storage":"10Gi","memory":"3Gi"}` | DataChain worker Job requests configuration |
+| studioDatachainWorkerJobTemplate.runtimeClassName | string | `nil` | DataChain worker Job security context configuration |
+| studioDatachainWorkerJobTemplate.securityContext | object | `{}` | DataChain worker Job security context configuration |
+| studioDatachainWorkerJobTemplate.tolerations | list | `[]` | DataChain worker Job pod tolerations configuration |
+| studioDatachainWorkerJobTemplate.ttlSecondsAfterFinished | int | `5` | DataChain worker Job TTL after finished (seconds) |
 | studioUi | object | `{"affinity":{},"autoscaling":{"enabled":false,"maxReplicas":5,"minReplicas":1,"targetCPUUtilizationPercentage":80},"envFromSecret":"","envVars":{},"image":{"pullPolicy":"IfNotPresent","repository":"docker.iterative.ai/studio-frontend"},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"memory":"2Gi"},"requests":{"cpu":"500m","memory":"1Gi"}},"securityContext":{},"service":{"port":3000,"type":"ClusterIP"},"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}},"tolerations":[]}` | Studio UI settings group |
 | studioUi.envFromSecret | string | `""` | The name of an existing Secret that contains sensitive environment variables passed to UI pods. |
 | studioUi.envVars | object | `{}` | Additional environment variables for ui pods |

charts/studio/templates/deployment-studio-datachain-worker.yaml

@@ -48,7 +48,7 @@ spec:
             {{- toYaml .Values.studioDatachainWorker.securityContext | nindent 12 }}
           image: "{{ .Values.studioDatachainWorker.image.repository }}:{{ .Values.studioDatachainWorker.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.studioDatachainWorker.image.pullPolicy }}
-          args: ["/app/bin/run_celery_worker_udf.sh"]
+          args: ["/app/bin/run_celery.sh", "datachain-master"]
           resources:
             {{- toYaml .Values.studioDatachainWorker.resources | nindent 12 }}
           env:
@@ -72,76 +72,12 @@ spec:
                 name: {{ .Values.studioDatachainWorker.envFromSecret }}
             {{- end }}
           volumeMounts:
-            {{- if not .Values.global.blobvault.bucket }}
-            - name: blobvault
-              mountPath: /blobvault
-            {{- end }}
             - name: studio-ca-certificates
               mountPath: /usr/local/share/ca-certificates
-            - name: tmp-ephemeral
-              mountPath: /tmp/shared
-            {{- if .Values.studioDatachainWorker.localStorage }}
-            - name: tmp-local
-              mountPath: /tmp/local
-            {{- end }}
       volumes:
-        {{- if not .Values.global.blobvault.bucket }}
-        - name: blobvault
-          persistentVolumeClaim:
-            claimName: blobvault
-        {{- end }}
         - name: studio-ca-certificates
           configMap:
             name: studio-ca-certificates
-        - name: tmp-ephemeral
-          {{- if eq .Values.studioDatachainWorker.ephemeralStorage.type "ephemeral" }}
-          ephemeral:
-            volumeClaimTemplate:
-              metadata:
-                labels:
-                  type: {{.Release.Name}}-{{ .Values.studioDatachainWorker.ephemeralStorage.persistentVolumeClaim.claimName }}
-              spec:
-                accessModes: [ "ReadWriteOnce" ]
-                {{- if .Values.studioDatachainWorker.ephemeralStorage.persistentVolumeClaim.storageClass }}
-                storageClassName: {{ .Values.studioDatachainWorker.ephemeralStorage.persistentVolumeClaim.storageClass }}
-                {{- end }}
-                resources:
-                  requests:
-                    storage: {{ .Values.studioDatachainWorker.ephemeralStorage.size }}
-          {{- else if eq .Values.studioDatachainWorker.ephemeralStorage.type "pvc" }}
-          persistentVolumeClaim:
-              claimName: {{ .Values.studioDatachainWorker.ephemeralStorage.persistentVolumeClaim.claimName }}
-          {{- else if eq .Values.studioDatachainWorker.ephemeralStorage.type "pvcRWX" }}
-          persistentVolumeClaim:
-              claimName: {{.Release.Name}}-studio-datachain-worker-ephemeral-rwx
-          {{- else if eq .Values.studioDatachainWorker.ephemeralStorage.type "emptyDir"}}
-          emptyDir:
-            sizeLimit: {{ .Values.studioDatachainWorker.ephemeralStorage.size }}
-          {{- end }}
-        {{- with .Values.studioDatachainWorker.localStorage }}
-        - name: tmp-local
-          {{- if eq .type "ephemeral" }}
-          ephemeral:
-            volumeClaimTemplate:
-              metadata:
-                labels:
-                  type: {{ $.Release.Name }}-{{ .persistentVolumeClaim.claimName }}
-              spec:
-                accessModes: [ "ReadWriteOnce" ]
-                {{- if .persistentVolumeClaim.storageClass }}
-                storageClassName: {{ .persistentVolumeClaim.storageClass }}
-                {{- end }}
-                resources:
-                  requests:
-                    storage: {{ .size }}
-          {{- else if eq .type "pvc" }}
-          persistentVolumeClaim:
-              claimName: {{ .persistentVolumeClaim.claimName }}
-          {{- else if eq .type "emptyDir"}}
-          emptyDir:
-            sizeLimit: {{ .size }}
-          {{- end }}
-        {{- end }}
       {{- with .Values.studioDatachainWorker.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}