Added singleuser.tolerations, for node taints

<core|user> pods now get `tolerations` for the node taint
`hub.jupyter.org/dedicated=<user|core>:NoSchedule` that could optionally
be added to nodes or all nodes in a node group (aka. node pool).

Note that due to a bug with using the `gcloud` CLI, we also add the
toleration for the same taint where `/` is replaced with `_`.

In this commit, `singleuser.extraTolerations` are now also made
configurable allowing you to add your own custom tolerations to the
user pods.

images/hub/jupyterhub_config.py

@@ -68,6 +68,7 @@
 c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {})
 
 c.KubeSpawner.image_spec = get_config('singleuser.image-spec')
+c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations', []))
 # Configure dynamically provisioning pvc
 storage_type = get_config('singleuser.storage.type')
 if storage_type == 'dynamic':

jupyterhub/schema.yaml

@@ -460,6 +460,22 @@ properties:
         description: |
           Deprecated and no longer does anything. Use the user-scheduler instead
           in order to accomplish a good packing of the user pods.
+      extraTolerations:
+        type: list
+        description: |
+          Tolerations allow a pod to be scheduled on nodes with taints. These
+          are additional tolerations other than the user pods and core pods
+          default ones `hub.jupyter.org/dedicated=user:NoSchedule` or
+          `hub.jupyter.org/dedicated=core:NoSchedule`. Note that a duplicate set
+          of tolerations exist where `/` is replaced with `_` as the Google
+          cloud does not support the character `/` yet in the toleration.
+
+          See the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)
+          for more info.
+
+          Pass this field an array of
+          [`Toleration`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core)
+          objects.
 
 
   scheduling:

jupyterhub/templates/hub/configmap.yaml

@@ -221,6 +221,9 @@ data:
     {{- end }}
   {{- end }}
 
+  singleuser.tolerations: |
+    {{- include "jupyterhub.userTolerations" . | nindent 4 }}
+
   {{- if .Values.scheduling.userScheduler.enabled }}
   singleuser.scheduler-name: "{{ .Release.Name }}-user-scheduler"
   {{- end }}

jupyterhub/templates/image-puller/_daemonset-helper.yaml

@@ -36,6 +36,8 @@ spec:
         {{- /* Changes here will cause the DaemonSet to restart the pods. */}}
         {{- include "jupyterhub.matchLabels" . | nindent 8 }}
     spec:
+      tolerations:
+        {{- include "jupyterhub.userTolerations" . | nindent 8 }}
       terminationGracePeriodSeconds: 0
       automountServiceAccountToken: false
       {{- if .Values.singleuser.imagePullSecret.enabled }}

jupyterhub/templates/scheduling/_scheduling-helpers.tpl

@@ -0,0 +1,17 @@
+{{- /*
+  jupyterhub.userTolerations
+    Lists the tolerations for node taints that the user pods should have
+*/}}
+{{- define "jupyterhub.userTolerations" -}}
+- key: hub.jupyter.org_dedicated
+  operator: Equal
+  value: user
+  effect: NoSchedule
+- key: hub.jupyter.org/dedicated
+  operator: Equal
+  value: user
+  effect: NoSchedule
+{{- if .Values.singleuser.extraTolerations -}}
+{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}

jupyterhub/values.yaml

@@ -142,6 +142,7 @@ auth:
 
 
 singleuser:
+  extraTolerations: []
   networkTools:
     image:
       name: jupyterhub/k8s-network-tools

tools/templates/lint-and-validate-values.yaml

@@ -149,7 +149,11 @@ auth:
 singleuser:
   nodeSelector:
     mock-node-selector: mock
-  extraTolerations: []
+  extraTolerations:
+    - key: hub.jupyter.org/test
+      operator: Equal
+      value: test
+      effect: NoSchedule
   extraNodeAffinity:
     required: []
     preferred: []