diff --git a/infrastructure/cluster/flux-v2/dask/README.md b/infrastructure/cluster/flux-v2/dask/README.md new file mode 100644 index 00000000..7fd195f8 --- /dev/null +++ b/infrastructure/cluster/flux-v2/dask/README.md @@ -0,0 +1,32 @@ +This deploymewnt allows multiple users to spawn multiple dask clusters by creating a scheduler each time a user asks for it, following the logic o fthe [Dask Gateway](https://gateway.dask.org/). +The deployment is done on top of the `jhub` namespace where the deployment of the Jupyterhub service is. The Dask packages are available inthe image called Dask at notebooks' start up. +If you decide to select this image, you can run a notebook and check that you can spawn a cluster by running: + +``` +import dask +import dask.distributed +from dask_gateway import Gateway +from dask_gateway import GatewayCluster +import dask.array as da +import requests +from dask_gateway import scheduler_preload +import os +``` + +To check the service of the `dask-gateway` is reachable, run: + +``` +requests.get("http://traefik-dask-dask-gateway/api/health").content +``` + +And then connect to the Gateway: + +``` +gateway = Gateway(address="http://traefik-dask-dask-gateway/", + auth="jupyterhub") +gateway.list_clusters() +cluster = gateway.new_cluster() +cluster.scale(4) # to create 4 worker nodes +``` + +You will see the scheduler and the worker pods appearing in the k8s cluster under the `jhub` namespace. \ No newline at end of file diff --git a/infrastructure/cluster/flux-v2/dask/daskhub-charts.yaml b/infrastructure/cluster/flux-v2/dask/dask-gateway-chart.yaml similarity index 57% rename from infrastructure/cluster/flux-v2/dask/daskhub-charts.yaml rename to infrastructure/cluster/flux-v2/dask/dask-gateway-chart.yaml index dbf4b48f..40f596ac 100644 --- a/infrastructure/cluster/flux-v2/dask/daskhub-charts.yaml +++ b/infrastructure/cluster/flux-v2/dask/dask-gateway-chart.yaml @@ -1,8 +1,9 @@ +# apiVersion: source.toolkit.fluxcd.io/v2beta1 apiVersion: source.toolkit.fluxcd.io/v1beta1 kind: HelmRepository metadata: - name: daskhub-charts - namespace: daskhub + name: dask-gateway-charts + namespace: jhub spec: interval: 10m url: https://helm.dask.org/ diff --git a/infrastructure/cluster/flux-v2/dask/dask-gateway-release.yaml b/infrastructure/cluster/flux-v2/dask/dask-gateway-release.yaml new file mode 100644 index 00000000..5399429a --- /dev/null +++ b/infrastructure/cluster/flux-v2/dask/dask-gateway-release.yaml @@ -0,0 +1,59 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: dask-gateway + namespace: jhub + annotations: + flux.weave.works/automated: "false" + +spec: + releaseName: dask + interval: 5m + chart: + spec: + sourceRef: + kind: HelmRepository + name: dask-gateway-charts + namespace: jhub + interval: 1m + chart: dask-gateway + # version: 2023.1.0 # support for python 3.11 + version: 2022.4.0 # support for python 3.9 + + valuesFrom: + - kind: Secret + name: api-dask-dask-gateway + valuesKey: jupyterhub-api-token + targetPath: gateway.auth.jupyterhub.apiToken + + + values: + + gateway: + # extraConfig: + # clusteroptions: | + # from dask_gateway_server.options import Options, Integer, Float, String + + # def option_handler(options): + # return { + # "worker_cores": options.worker_cores, + # "worker_memory": "%fG" % options.worker_memory, + # "image": options.image, + # } + + # c.Backend.cluster_options = Options( + # Integer("worker_cores", 2, min=1, max=4, label="Worker Cores"), + # Float("worker_memory", 4, min=1, max=8, label="Worker Memory (GiB)"), + # String("image", default="ghcr.io/dask/dask-gateway:2022.4.0", label="Image"), + # handler=option_handler, + # ) + + # Number of instances of the gateway-server to run + replicas: 1 + + # prefix: "/services/dask-gateway" + + auth: + type: jupyterhub + + \ No newline at end of file diff --git a/infrastructure/cluster/flux-v2/dask/daskhub-namespace.yaml b/infrastructure/cluster/flux-v2/dask/daskhub-namespace.yaml deleted file mode 100644 index 4cbb4700..00000000 --- a/infrastructure/cluster/flux-v2/dask/daskhub-namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: daskhub - labels: - name: daskhub \ No newline at end of file diff --git a/infrastructure/cluster/flux-v2/dask/daskhub-pvc.yaml b/infrastructure/cluster/flux-v2/dask/daskhub-pvc.yaml deleted file mode 100644 index 51c962de..00000000 --- a/infrastructure/cluster/flux-v2/dask/daskhub-pvc.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: daskhub-vre-dynamic - namespace: daskhub -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 500Gi - storageClassName: manila-meyrin-cephfs-retain - \ No newline at end of file diff --git a/infrastructure/cluster/flux-v2/dask/daskhub-release.yaml b/infrastructure/cluster/flux-v2/dask/daskhub-release.yaml deleted file mode 100644 index aef82659..00000000 --- a/infrastructure/cluster/flux-v2/dask/daskhub-release.yaml +++ /dev/null @@ -1,206 +0,0 @@ -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: daskhub -# namespace: daskhub -# annotations: -# flux.weave.works/automated: "false" - -# spec: -# releaseName: dask -# interval: 5m -# chart: -# spec: -# sourceRef: -# kind: HelmRepository -# name: daskhub-charts -# namespace: daskhub -# interval: 1m -# chart: daskhub -# version: 2022.11.0 -# # version: 2023.1.0 - -# valuesFrom: -# - kind: Secret -# name: nb-vre-iam-client -# valuesKey: client_id -# targetPath: jupyterhub.hub.config.GenericOAuthenticator.client_id -# - kind: Secret -# name: nb-vre-iam-client -# valuesKey: client_secret -# targetPath: jupyterhub.hub.config.GenericOAuthenticator.client_secret -# - kind: Secret -# name: nb-vre-api-token -# valuesKey: apiToken -# targetPath: hub.services.dask-gateway.apiToken -# - kind: Secret -# name: nb-vre-api-token -# valuesKey: apiToken -# targetPath: dask-gateway.gateway.auth.jupyterhub.apiToken -# - kind: Secret -# name: daskhub-vre-dbconnectstring -# valuesKey: dbconnectstring -# targetPath: jupyterhub.hub.db.url - -# values: - -# rbac: -# enabled: true # Create and use roles and service accounts on an RBAC-enabled cluster. - -# jupyterhub: - -# proxy: -# # run only on master nodes -# chp: -# nodeSelector: -# node-role.kubernetes.io/master: "" -# tolerations: -# - key: "node-role.kubernetes.io/master" -# operator: Exists - -# hub: - -# networkPolicy: -# enabled: false -# db: -# type: postgres - -# # type: sqlite-memory -# # upgrade: true - -# config: -# GenericOAuthenticator: -# oauth_callback_url: https://nb-vre.cern.ch/hub/oauth_callback -# authorize_url: https://iam-escape.cloud.cnaf.infn.it/authorize -# token_url: https://iam-escape.cloud.cnaf.infn.it/token -# userdata_url: https://iam-escape.cloud.cnaf.infn.it/userinfo -# scope: -# - openid -# - profile -# - email -# - offline_access -# username_key: preferred_username -# JupyterHub: -# authenticator_class: generic-oauth - -# # run only on master nodes -# nodeSelector: -# node-role.kubernetes.io/master: "" -# tolerations: -# - key: "node-role.kubernetes.io/master" -# operator: Exists - -# extraConfig: -# # Register Dask Gateway service and setup singleuser environment. -# 00-add-dask-gateway-values: | -# # 1. Sets `DASK_GATEWAY__PROXY_ADDRESS` in the singleuser environment. -# # 2. Adds the URL for the Dask Gateway JupyterHub service. -# import os - -# # These are set by jupyterhub. -# release_name = os.environ["HELM_RELEASE_NAME"] -# release_namespace = os.environ["POD_NAMESPACE"] - -# if "PROXY_HTTP_SERVICE_HOST" in os.environ: -# # https is enabled, we want to use the internal http service. -# gateway_address = "http://{}:{}/services/dask-gateway/".format( -# os.environ["PROXY_HTTP_SERVICE_HOST"], -# os.environ["PROXY_HTTP_SERVICE_PORT"], -# ) -# print("Setting DASK_GATEWAY__ADDRESS {} from HTTP service".format(gateway_address)) -# else: -# gateway_address = "http://proxy-public/services/dask-gateway" -# print("Setting DASK_GATEWAY__ADDRESS {}".format(gateway_address)) - -# # Internal address to connect to the Dask Gateway. -# c.KubeSpawner.environment.setdefault("DASK_GATEWAY__ADDRESS", gateway_address) -# # Internal address for the Dask Gateway proxy. -# c.KubeSpawner.environment.setdefault("DASK_GATEWAY__PROXY_ADDRESS", "gateway://traefik-{}-dask-gateway.{}:80".format(release_name, release_namespace)) -# # Relative address for the dashboard link. -# c.KubeSpawner.environment.setdefault("DASK_GATEWAY__PUBLIC_ADDRESS", "/services/dask-gateway/") -# # Use JupyterHub to authenticate with Dask Gateway. -# c.KubeSpawner.environment.setdefault("DASK_GATEWAY__AUTH__TYPE", "jupyterhub") - -# # Adds Dask Gateway as a JupyterHub service to make the gateway available at -# # {HUB_URL}/services/dask-gateway -# service_url = "http://traefik-{}-dask-gateway.{}".format(release_name, release_namespace) -# for service in c.JupyterHub.services: -# if service["name"] == "dask-gateway": -# if not service.get("url", None): -# print("Adding dask-gateway service URL") -# service.setdefault("url", service_url) -# break -# else: -# print("dask-gateway service not found. Did you set jupyterhub.hub.services.dask-gateway.apiToken?") - -# singleuser: - -# # image: -# # name: ghcr.io/egazzarr/docker-tests-dask-root -# # # name: dask-root # Image to use for singleuser environment. Must include dask-gateway. -# # tag: latest -# # pullPolicy: Always - -# defaultUrl: "/lab" # Use jupyterlab by defualt. - -# memory: -# guarantee: 500M -# limit: 1G -# cpu: -# guarantee: 0.3 -# limit: 1 -# # storage: -# # type: static -# # static: -# # pvcName: daskhub-vre-dynamic - -# # storage: -# # type: dynamic -# # capacity: 1Gi -# # dynamic: -# # storageClass: standard -# # default is 300s, sometimes Jetstream volumes are slow to attach -# startTimeout: 600 -# # See https://github.com/zonca/jupyterhub-deploy-kubernetes-jetstream/issues/38 -# lifecycleHooks: -# postStart: -# exec: -# command: -# - "sh" -# - "-c" -# - > -# chmod 700 .ssh; -# chmod g-s .ssh; -# chmod 600 .ssh/*; -# exit 0 -# ingress: -# enabled: true -# ingressClassName: nginx -# annotations: -# cert-manager.io/cluster-issuer: "letsencrypt" # this issues a certificate for the domain through cert-manager automatically, and creates the secret in ingress.tls.hosts.secretName -# hosts: -# - nb-vre.cern.ch -# tls: -# - hosts: -# - nb-vre.cern.ch -# secretName: cert-manager-tls-ingress-secret-nb - -# dask-gateway: -# enabled: true # Enabling dask-gateway will install Dask Gateway as a dependency. -# # Futher Dask Gateway configuration goes here -# # See https://github.com/dask/dask-gateway/blob/master/resources/helm/dask-gateway/values.yaml -# gateway: -# prefix: "/services/dask-gateway" # Users connect to the Gateway through the JupyterHub service. -# auth: -# type: jupyterhub # Use JupyterHub to authenticate with Dask Gateway -# traefik: -# service: -# type: LoadBalancer # Access Dask Gateway through JupyterHub. To access the Gateway from outside JupyterHub, this must be changed to a `LoadBalancer`. - -# dask-kubernetes: -# # Use dask-kubernetes, rather than Dask Gateway, for creating Dask Clusters. -# # Enabling this also requires -# # 1. Setting `jupyterhub.singleuser.serviceAccountName: daskkubernetes`. -# # 2. Ensuring that `dask-kubernetes` is in your singleuser environment. -# enabled: false -