destination-earth · minrk · Sep 20, 2024 · Sep 18, 2024 · Sep 20, 2024
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -38,7 +38,7 @@ repos:
     hooks:
       - id: chartpress
         # run in subdirectory
-        entry: bash -c 'cd gfts-track-reconstruction/jupyterhub && "$@"' --
+        entry: bash -c 'cd gfts-track-reconstruction/jupyterhub && chartpress --reset'
 
   # Autoformat and linting, misc. details
   - repo: https://github.com/pre-commit/pre-commit-hooks

diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock
@@ -5,6 +5,9 @@ dependencies:
 - name: dask-gateway
   repository: https://helm.dask.org/
   version: 2024.1.0
+- name: kbatch-proxy
+  repository: https://kbatch-dev.github.io/helm-chart
+  version: 0.4.2
 - name: ingress-nginx
   repository: https://kubernetes.github.io/ingress-nginx
   version: 4.9.1
@@ -14,5 +17,5 @@ dependencies:
 - name: grafana
   repository: https://grafana.github.io/helm-charts
   version: 7.0.14
-digest: sha256:f65f939aed209d2ebec4ea4534deef0d14d8b99ee1c5ee20ddee11d0a7115b20
-generated: "2024-04-03T14:18:11.909303+02:00"
+digest: sha256:537e964a33edb2d34d66718427bc2b0716efc7c299330414696b1e80eb3f91ac
+generated: "2024-09-04T14:58:24.438886+02:00"
diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml
@@ -14,6 +14,11 @@ dependencies:
     version: "2024.1.0"
     repository: https://helm.dask.org/
 
+  # kbatch
+  - name: kbatch-proxy
+    version: "0.4.2"
+    repository: https://kbatch-dev.github.io/helm-chart
+
   # ingress
   - name: ingress-nginx
     version: "4.9.1"

diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml
@@ -0,0 +1,31 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kbatch-role
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["create", "get", "list", "delete"]
+
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "configmaps"]
+    verbs: ["get", "watch", "list", "create", "delete", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kbatch
+subjects:
+  - kind: User
+    name: system:serviceaccount:{{ .Release.Namespace }}:kbatch-proxy
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: kbatch-role
+  apiGroup: rbac.authorization.k8s.io
diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml
@@ -30,6 +30,16 @@ jupyterhub:
       name: "c63eqfuv.c1.gra9.container-registry.ovh.net/gfts/jupyterhub-user"
       tag: "set-by-chartpress"
       pullPolicy: Always
+    networkPolicy:
+      egress:
+        # allow access to kbatch-proxy
+        - to:
+            - podSelector:
+                matchLabels:
+                  app.kubernetes.io/name: kbatch-proxy
+          ports:
+            - protocol: TCP
+              port: 80
     memory:
       limit: 24G
       guarantee: 8G
@@ -43,6 +53,7 @@ jupyterhub:
       CULL_INTERVAL: "120"
       GH_SCOPED_CREDS_CLIENT_ID: Iv1.f4a7db20c671f599
       GH_SCOPED_CREDS_APP_URL: https://github.com/apps/gfts-jupyterhub
+      KBATCH_URL: http://kbatch-proxy
       AWS_ENDPOINT_URL_S3: "https://s3.gra.perf.cloud.ovh.net"
       AWS_DEFAULT_REGION: gra
       # JUPYTER_FS_BUCKETS: destine-gfts-data-lake,gfts-reference-data,gfts-ifremer
@@ -186,6 +197,11 @@ jupyterhub:
 
         c.KubeSpawner.pre_spawn_hook = load_creds
     loadRoles:
+      server:
+        scopes:
+          - access:servers!server
+          - users:activity!user
+          - access:services!service=kbatch
       management:
         scopes:
           - admin-ui
@@ -197,6 +213,13 @@ jupyterhub:
         users:
           - annefou
           - minrk
+      kbatch-users:
+        scopes:
+          - "access:services!service=kbatch"
+        users:
+          - minrk
+    services:
+      kbatch-proxy: {}
 
   scheduling:
     userScheduler:
@@ -314,3 +337,92 @@ dask-gateway:
 
         # shutdown idle clusters after one hour
         c.ClusterConfig.idle_timeout = 3600
+
+kbatch-proxy:
+  fullnameOverride: kbatch-proxy
+  image:
+    repository: ghcr.io/minrk/kbatch-proxy
+    tag: "gfts"
+    pullPolicy: Always
+  app:
+    # jupyterhub_service_prefix: /services/kbatch-proxy
+    # cannot use internal ip
+    # pending https://github.com/kbatch-dev/helm-chart/pull/6
+    jupyterhub_api_url: https://gfts.minrk.net/hub/api/
+    extra_env:
+      JUPYTERHUB_SERVICE_NAME: kbatch
+      KBATCH_JOB_TEMPLATE_FILE: /srv/job_template.yaml
+      KBATCH_PROFILE_FILE: /srv/profiles.yaml
+    extraFiles:
+      job_template:
+        mountPath: /srv/job_template.yaml
+        data:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            labels:
+              app.kubernetes.io/managed-by: kbatch
+          spec:
+            template:
+              metadata:
+                labels:
+                  app.kubernetes.io/managed-by: kbatch
+              spec:
+                affinity:
+                  nodeAffinity:
+                    requiredDuringSchedulingIgnoredDuringExecution:
+                      nodeSelectorTerms:
+                        - matchExpressions:
+                            - key: hub.jupyter.org/node-purpose
+                              operator: In
+                              values:
+                                - user
+      profiles:
+        mountPath: /srv/profiles.yaml
+        data:
+          default:
+            resources:
+              requests:
+                cpu: "2"
+                memory: "8G"
+              limits:
+                cpu: "2"
+                memory: "8G"
+          big60:
+            resources:
+              requests:
+                cpu: "7"
+                memory: "60G"
+              limits:
+                cpu: "8"
+                memory: "60G"
+          big120:
+            resources:
+              requests:
+                cpu: "14"
+                memory: "120G"
+              limits:
+                cpu: "15"
+                memory: "120G"
+          big240:
+            resources:
+              requests:
+                cpu: "29"
+                memory: "240G"
+              limits:
+                cpu: "30"
+                memory: "240G"
+          huge:
+            resources:
+              requests:
+                cpu: "59"
+                memory: "480G"
+              limits:
+                cpu: "60"
+                memory: "480G"
+            node_affinity_required:
+              - matchExpressions:
+                  - key: gfts.destination-earth.eu/size
+                    operator: In
+                    values:
+                      - big512
diff --git a/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt b/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt
@@ -6,6 +6,7 @@ gh-scoped-creds
 isort
 jupyter-keepalive
 jupyterhub==4.1.6
+kbatch @ git+https://github.com/kbatch-dev/kbatch#subdirectory=kbatch
 # git+https://github.com/iaocea/pangeo-fish#egg=pangeo-fish
 #git+https://github.com/iaocea/xarray-regridding#egg=xarray-regridding
 xarray-healpy @ git+https://github.com/iaocea/xarray-regridding.git@0ffca6058f4008f4f22f076e2d60787fcf32ac82
diff --git a/gfts-track-reconstruction/jupyterhub/secrets/config.yaml b/gfts-track-reconstruction/jupyterhub/secrets/config.yaml