From a95873b17309c5ca71642b68b34fa6a3698e61e1 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sun, 9 Sep 2018 02:12:25 +0200 Subject: [PATCH 1/2] Storage labels: configurable extras Extra storage labels are now configurable through config.yaml and `singleuser.storageExtraLabels`. --- images/hub/jupyterhub_config.py | 1 + jupyterhub/templates/hub/configmap.yaml | 6 ++++++ jupyterhub/values.yaml | 1 + tools/templates/lint-and-validate-values.yaml | 3 ++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/images/hub/jupyterhub_config.py b/images/hub/jupyterhub_config.py index f9cae389c4..d70ea97981 100644 --- a/images/hub/jupyterhub_config.py +++ b/images/hub/jupyterhub_config.py @@ -65,6 +65,7 @@ ('node_selector', 'node-selector'), ): set_config_if_not_none(c.KubeSpawner, trait, 'singleuser.' + cfg_key) +c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {}) c.KubeSpawner.image_spec = get_config('singleuser.image-spec') # Configure dynamically provisioning pvc diff --git a/jupyterhub/templates/hub/configmap.yaml b/jupyterhub/templates/hub/configmap.yaml index 3f8bd1eaa2..a51e4ae6be 100644 --- a/jupyterhub/templates/hub/configmap.yaml +++ b/jupyterhub/templates/hub/configmap.yaml @@ -208,6 +208,12 @@ data: {{- range $key, $value := .Values.singleuser.extraLabels }} {{ $key | quote }}: {{ $value | quote }} {{- end }} + {{- if .Values.singleuser.storage.extraLabels }} + singleuser.storage-extra-labels: | + {{- range $key, $value := .Values.singleuser.storage.extraLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- end }} {{- if .Values.singleuser.extraEnv }} singleuser.extra-env: | {{- range $key, $value := .Values.singleuser.extraEnv }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index eec2bfac2b..ce4c58e752 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -171,6 +171,7 @@ singleuser: serviceAccountName: storage: type: dynamic + extraLabels: {} extraVolumes: [] extraVolumeMounts: [] static: diff --git a/tools/templates/lint-and-validate-values.yaml b/tools/templates/lint-and-validate-values.yaml index 8552b90502..9eea1310d5 100644 --- a/tools/templates/lint-and-validate-values.yaml +++ b/tools/templates/lint-and-validate-values.yaml @@ -172,7 +172,6 @@ singleuser: - 169.254.169.254/32 events: true extraLabels: {} - storageExtraLabels: {} extraEnv: {} lifecycleHooks: initContainers: @@ -188,6 +187,8 @@ singleuser: serviceAccountName: storage: type: dynamic + extraLabels: + mock-label: mock-value extraVolumes: [] extraVolumeMounts: [] static: From f243103627235c6a786bcbd1ec64698576bfb368 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 11 Sep 2018 20:08:44 +0200 Subject: [PATCH 2/2] Added `singleuser.tolerations`, for node taints pods now get `tolerations` for the node taint `hub.jupyter.org/dedicated=:NoSchedule` that could optionally be added to nodes or all nodes in a node group (aka. node pool). Note that due to a bug with using the `gcloud` CLI, we also add the toleration for the same taint where `/` is replaced with `_`. In this commit, `singleuser.extraTolerations` are now also made configurable allowing you to add your own custom tolerations to the user pods. --- images/hub/jupyterhub_config.py | 1 + jupyterhub/schema.yaml | 16 ++++++++++++++++ jupyterhub/templates/hub/configmap.yaml | 3 +++ .../image-puller/_daemonset-helper.yaml | 2 ++ .../scheduling/_scheduling-helpers.tpl | 17 +++++++++++++++++ jupyterhub/values.yaml | 1 + tools/templates/lint-and-validate-values.yaml | 6 +++++- 7 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 jupyterhub/templates/scheduling/_scheduling-helpers.tpl diff --git a/images/hub/jupyterhub_config.py b/images/hub/jupyterhub_config.py index d70ea97981..b06f708bf2 100644 --- a/images/hub/jupyterhub_config.py +++ b/images/hub/jupyterhub_config.py @@ -68,6 +68,7 @@ c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {}) c.KubeSpawner.image_spec = get_config('singleuser.image-spec') +c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations', [])) # Configure dynamically provisioning pvc storage_type = get_config('singleuser.storage.type') if storage_type == 'dynamic': diff --git a/jupyterhub/schema.yaml b/jupyterhub/schema.yaml index dd7baacd04..0007abe40b 100644 --- a/jupyterhub/schema.yaml +++ b/jupyterhub/schema.yaml @@ -460,6 +460,22 @@ properties: description: | Deprecated and no longer does anything. Use the user-scheduler instead in order to accomplish a good packing of the user pods. + extraTolerations: + type: list + description: | + Tolerations allow a pod to be scheduled on nodes with taints. These + are additional tolerations other than the user pods and core pods + default ones `hub.jupyter.org/dedicated=user:NoSchedule` or + `hub.jupyter.org/dedicated=core:NoSchedule`. Note that a duplicate set + of tolerations exist where `/` is replaced with `_` as the Google + cloud does not support the character `/` yet in the toleration. + + See the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + for more info. + + Pass this field an array of + [`Toleration`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core) + objects. scheduling: diff --git a/jupyterhub/templates/hub/configmap.yaml b/jupyterhub/templates/hub/configmap.yaml index a51e4ae6be..25f05951f3 100644 --- a/jupyterhub/templates/hub/configmap.yaml +++ b/jupyterhub/templates/hub/configmap.yaml @@ -221,6 +221,9 @@ data: {{- end }} {{- end }} + singleuser.tolerations: | + {{- include "jupyterhub.userTolerations" . | nindent 4 }} + {{- if .Values.scheduling.userScheduler.enabled }} singleuser.scheduler-name: "{{ .Release.Name }}-user-scheduler" {{- end }} diff --git a/jupyterhub/templates/image-puller/_daemonset-helper.yaml b/jupyterhub/templates/image-puller/_daemonset-helper.yaml index ed0ec4e522..a41c7ede54 100644 --- a/jupyterhub/templates/image-puller/_daemonset-helper.yaml +++ b/jupyterhub/templates/image-puller/_daemonset-helper.yaml @@ -36,6 +36,8 @@ spec: {{- /* Changes here will cause the DaemonSet to restart the pods. */}} {{- include "jupyterhub.matchLabels" . | nindent 8 }} spec: + tolerations: + {{- include "jupyterhub.userTolerations" . | nindent 8 }} terminationGracePeriodSeconds: 0 automountServiceAccountToken: false {{- if .Values.singleuser.imagePullSecret.enabled }} diff --git a/jupyterhub/templates/scheduling/_scheduling-helpers.tpl b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl new file mode 100644 index 0000000000..2aff62b997 --- /dev/null +++ b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl @@ -0,0 +1,17 @@ +{{- /* + jupyterhub.userTolerations + Lists the tolerations for node taints that the user pods should have +*/}} +{{- define "jupyterhub.userTolerations" -}} +- key: hub.jupyter.org_dedicated + operator: Equal + value: user + effect: NoSchedule +- key: hub.jupyter.org/dedicated + operator: Equal + value: user + effect: NoSchedule +{{- if .Values.singleuser.extraTolerations }} +{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index ce4c58e752..823fe2fc10 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -142,6 +142,7 @@ auth: singleuser: + extraTolerations: [] networkTools: image: name: jupyterhub/k8s-network-tools diff --git a/tools/templates/lint-and-validate-values.yaml b/tools/templates/lint-and-validate-values.yaml index 9eea1310d5..41163b5ca6 100644 --- a/tools/templates/lint-and-validate-values.yaml +++ b/tools/templates/lint-and-validate-values.yaml @@ -149,7 +149,11 @@ auth: singleuser: nodeSelector: mock-node-selector: mock - extraTolerations: [] + extraTolerations: + - key: hub.jupyter.org/test + operator: Equal + value: test + effect: NoSchedule extraNodeAffinity: required: [] preferred: []