[Part 9 of #758] Preset: scheduling.nodeAffinity.matchNodePurpose

images/hub/jupyterhub_config.py

@@ -62,11 +62,19 @@
     ('fs_gid', 'fs-gid'),
     ('service_account', 'service-account-name'),
     ('scheduler_name', 'scheduler-name'),
-    ('node_selector', 'node-selector'),
 ):
     set_config_if_not_none(c.KubeSpawner, trait, 'singleuser.' + cfg_key)
+c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {})
 
 c.KubeSpawner.image_spec = get_config('singleuser.image-spec')
+c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations', []))
+c.KubeSpawner.node_selector.update(get_config('singleuser.node-selector', {}))
+c.KubeSpawner.node_affinity_required.extend(get_config('singleuser.node-affinity-required', []))
+c.KubeSpawner.node_affinity_preferred.extend(get_config('singleuser.node-affinity-preferred', []))
+c.KubeSpawner.pod_affinity_required.extend(get_config('singleuser.pod-affinity-required', []))
+c.KubeSpawner.pod_affinity_preferred.extend(get_config('singleuser.pod-affinity-preferred', []))
+c.KubeSpawner.pod_anti_affinity_required.extend(get_config('singleuser.pod-anti-affinity-required', []))
+c.KubeSpawner.pod_anti_affinity_preferred.extend(get_config('singleuser.pod-anti-affinity-preferred', []))
 # Configure dynamically provisioning pvc
 storage_type = get_config('singleuser.storage.type')
 if storage_type == 'dynamic':

jupyterhub/schema.yaml

@@ -460,6 +460,81 @@ properties:
         description: |
           Deprecated and no longer does anything. Use the user-scheduler instead
           in order to accomplish a good packing of the user pods.
+      extraTolerations:
+        type: list
+        description: |
+          Tolerations allow a pod to be scheduled on nodes with taints. These
+          are additional tolerations other than the user pods and core pods
+          default ones `hub.jupyter.org/dedicated=user:NoSchedule` or
+          `hub.jupyter.org/dedicated=core:NoSchedule`. Note that a duplicate set
+          of tolerations exist where `/` is replaced with `_` as the Google
+          cloud does not support the character `/` yet in the toleration.
+
+          See the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)
+          for more info.
+
+          Pass this field an array of
+          [`Toleration`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core)
+          objects.
+      extraNodeAffinity:
+        type: object
+        description: |
+          Affinities describe where pods prefer or require to be scheduled, they
+          may prefer or require a node where they are to be scheduled to have a
+          certain label (node affinity). They may also require to be scheduled
+          in proximity or with a lack of proximity to another pod (pod affinity
+          and anti pod affinity).
+
+          See the [Kubernetes
+          docs](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)
+          for more info.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`NodeSelectorTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#nodeselectorterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PreferredSchedulingTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#preferredschedulingterm-v1-core)
+              objects.
+      extraPodAffinity:
+        type: object
+        description: |
+          See the description of `singleuser.extraNodeAffinity`.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core)
+              objects.
+      extraPodAntiAffinity:
+        type: object
+        description: |
+          See the description of `singleuser.extraNodeAffinity`.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core)
+              objects.
 
 
   scheduling:
@@ -495,3 +570,51 @@ properties:
                 type:
                   - string
                   - "null"
+      corePods:
+        type: object
+        description: |
+          These settings influence the core pods like the hub, proxy and
+          user-scheduler pods.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if core pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=core
+                  ```
+      userPods:
+        type: object
+        description: |
+          These settings influence the user pods like the user-placeholder,
+          user-dummy and actual user pods named like jupyter-someusername.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if user pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=user
+                  ```

jupyterhub/templates/hub/configmap.yaml

@@ -208,13 +208,47 @@ data:
     {{- range $key, $value := .Values.singleuser.extraLabels }}
     {{ $key | quote }}: {{ $value | quote }}
     {{- end }}
+  {{- if .Values.singleuser.storage.extraLabels }}
+  singleuser.storage-extra-labels: |
+    {{- range $key, $value := .Values.singleuser.storage.extraLabels }}
+    {{ $key | quote }}: {{ $value | quote }}
+    {{- end }}
+  {{- end }}
   {{- if .Values.singleuser.extraEnv }}
   singleuser.extra-env: |
     {{- range $key, $value := .Values.singleuser.extraEnv }}
     {{ $key | quote }}: {{ $value | quote }}
     {{- end }}
   {{- end }}
 
+  singleuser.tolerations: |
+    {{- include "jupyterhub.userTolerations" . | nindent 4 }}
+
+  {{- if include "jupyterhub.userNodeAffinityRequired" . }}
+  singleuser.node-affinity-required: |
+    {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 4 }}
+  {{- end }}
+  {{- if include "jupyterhub.userNodeAffinityPreferred" . }}
+  singleuser.node-affinity-preferred: |
+    {{- include "jupyterhub.userNodeAffinityPreferred" . | nindent 4 }}
+  {{- end }}
+  {{- if include "jupyterhub.userPodAffinityRequired" . }}
+  singleuser.pod-affinity-required: |
+    {{- include "jupyterhub.userPodAffinityRequired" . | nindent 4 }}
+  {{- end }}
+  {{- if include "jupyterhub.userPodAffinityPreferred" . }}
+  singleuser.pod-affinity-preferred: |
+    {{- include "jupyterhub.userPodAffinityPreferred" . | nindent 4 }}
+  {{- end }}
+  {{- if include "jupyterhub.userPodAntiAffinityRequired" . }}
+  singleuser.pod-anti-affinity-required: |
+    {{- include "jupyterhub.userPodAntiAffinityRequired" . | nindent 4 }}
+  {{- end }}
+  {{- if include "jupyterhub.userPodAntiAffinityPreferred" . }}
+  singleuser.pod-anti-affinity-preferred: |
+    {{- include "jupyterhub.userPodAntiAffinityPreferred" . | nindent 4 }}
+  {{- end }}
+
   {{- if .Values.scheduling.userScheduler.enabled }}
   singleuser.scheduler-name: "{{ .Release.Name }}-user-scheduler"
   {{- end }}

jupyterhub/templates/hub/deployment.yaml

@@ -31,20 +31,7 @@ spec:
         {{- end }}
     spec:
       nodeSelector: {{ toJson .Values.hub.nodeSelector }}
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['proxy']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
+      {{- include "jupyterhub.coreAffinity" . | nindent 6 }}
       volumes:
         - name: config
           configMap:

jupyterhub/templates/image-puller/_daemonset-helper.yaml

@@ -36,6 +36,16 @@ spec:
         {{- /* Changes here will cause the DaemonSet to restart the pods. */}}
         {{- include "jupyterhub.matchLabels" . | nindent 8 }}
     spec:
+      tolerations:
+        {{- include "jupyterhub.userTolerations" . | nindent 8 }}
+      nodeSelector: {{ toJson .Values.singleuser.nodeSelector }}
+      {{- if include "jupyterhub.userNodeAffinityRequired" . }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 14 }}
+      {{- end }}
       terminationGracePeriodSeconds: 0
       automountServiceAccountToken: false
       {{- if .Values.singleuser.imagePullSecret.enabled }}
@@ -77,7 +87,6 @@ spec:
             - -c
             - echo "Pulling complete"
         {{- end }}
-      nodeSelector: {{ toJson .Values.singleuser.nodeSelector }}
       containers:
         - name: pause
           image: {{ .Values.prePuller.pause.image.name }}:{{ .Values.prePuller.pause.image.tag }}

jupyterhub/templates/proxy/autohttps/deployment.yaml

@@ -35,22 +35,9 @@ spec:
       {{- if .Values.rbac.enabled }}
       serviceAccountName: autohttps
       {{- end }}
-      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
       terminationGracePeriodSeconds: 60
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['hub']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
+      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
+      {{- include "jupyterhub.coreAffinity" . | nindent 6 }}
       containers:
         - name: nginx
           image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}"

jupyterhub/templates/proxy/deployment.yaml

@@ -29,22 +29,9 @@ spec:
         {{- .Values.proxy.annotations | toYaml | trimSuffix "\n" | nindent 8 }}
         {{- end }}
     spec:
-      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
       terminationGracePeriodSeconds: 60
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['hub']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
+      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
+      {{- include "jupyterhub.coreAffinity" . | nindent 6 }}
       {{- if $manualHTTPSwithsecret }}
       volumes:
         - name: tls-secret

jupyterhub/templates/scheduling/_scheduling-helpers.tpl

@@ -0,0 +1,99 @@
+{{- /*
+  jupyterhub.userTolerations
+    Lists the tolerations for node taints that the user pods should have
+*/}}
+{{- define "jupyterhub.userTolerations" -}}
+- key: hub.jupyter.org_dedicated
+  operator: Equal
+  value: user
+  effect: NoSchedule
+- key: hub.jupyter.org/dedicated
+  operator: Equal
+  value: user
+  effect: NoSchedule
+{{- if .Values.singleuser.extraTolerations }}
+{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+
+
+{{- define "jupyterhub.userNodeAffinityRequired" -}}
+{{- if eq .Values.scheduling.userPods.nodeAffinity.matchNodePurpose "require" -}}
+- matchExpressions:
+  - key: hub.jupyter.org/node-purpose
+    operator: In
+    values: [user]
+    {{- if .Values.singleuser.extraNodeAffinity.required }}{{ println }}{{ end }}
+{{- end }}
+{{- if .Values.singleuser.extraNodeAffinity.required -}}
+{{ .Values.singleuser.extraNodeAffinity.required | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.userNodeAffinityPreferred" -}}
+{{- if eq .Values.scheduling.userPods.nodeAffinity.matchNodePurpose "prefer" -}}
+- weight: 100
+  preference:
+    matchExpressions:
+      - key: hub.jupyter.org/node-purpose
+        operator: In
+        values: [user]
+        {{- if .Values.singleuser.extraNodeAffinity.preferred }}{{ println }}{{ end }}
+{{- end }}
+{{- if .Values.singleuser.extraNodeAffinity.preferred -}}
+{{ .Values.singleuser.extraNodeAffinity.preferred | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.userPodAffinityRequired" -}}
+{{- if .Values.singleuser.extraPodAffinity.required -}}
+{{ .Values.singleuser.extraPodAffinity.required | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.userPodAffinityPreferred" -}}
+{{- if .Values.singleuser.extraPodAffinity.preferred -}}
+{{ .Values.singleuser.extraPodAffinity.preferred | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.userPodAntiAffinityRequired" -}}
+{{- if .Values.singleuser.extraPodAntiAffinity.required -}}
+{{ .Values.singleuser.extraPodAntiAffinity.required | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.userPodAntiAffinityPreferred" -}}
+{{- if .Values.singleuser.extraPodAntiAffinity.preferred -}}
+{{ .Values.singleuser.extraPodAntiAffinity.preferred | toYaml | trimSuffix "\n" }}
+{{- end }}
+{{- end }}
+
+
+
+{{- define "jupyterhub.coreAffinity" -}}
+{{- $require := eq .Values.scheduling.corePods.nodeAffinity.matchNodePurpose "require" -}}
+{{- $prefer := eq .Values.scheduling.corePods.nodeAffinity.matchNodePurpose "prefer" -}}
+{{- if or $require $prefer -}}
+affinity:
+  nodeAffinity:
+    {{- if $require }}
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+          - key: hub.jupyter.org/node-purpose
+            operator: In
+            values: [core]
+    {{- end }}
+    {{- if $prefer }}
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        preference:
+          matchExpressions:
+            - key: hub.jupyter.org/node-purpose
+              operator: In
+              values: [core]
+    {{- end }}
+{{- end }}
+{{- end }}