Added support for configuring extraTolerations and extra...Affinity

For use with the kubespawner features introduced in 
jupyterhub/kubespawner#205.

images/hub/jupyterhub_config.py

@@ -65,6 +65,14 @@
     c.KubeSpawner.service_account = service_account_name
 
 c.KubeSpawner.node_selector = get_config('singleuser.node-selector')
+
+c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations-list', []))
+c.KubeSpawner.node_affinity_required.extend(get_config('singleuser.node-affinity-required', []))
+c.KubeSpawner.node_affinity_preferred.extend(get_config('singleuser.node-affinity-preferred', []))
+c.KubeSpawner.pod_affinity_required.extend(get_config('singleuser.pod-affinity-required', []))
+c.KubeSpawner.pod_affinity_preferred.extend(get_config('singleuser.pod-affinity-preferred', []))
+c.KubeSpawner.pod_anti_affinity_required.extend(get_config('singleuser.pod-anti-affinity-required', []))
+c.KubeSpawner.pod_anti_affinity_preferred.extend(get_config('singleuser.pod-anti-affinity-preferred', []))
 # Configure dynamically provisioning pvc
 storage_type = get_config('singleuser.storage.type')
 if storage_type == 'dynamic':

jupyterhub/templates/hub/configmap.yaml

@@ -187,7 +187,37 @@ data:
     {{ $key | quote }}: {{ $value | quote }}
     {{- end }}
   {{- end }}
-
+
+  {{- $_ := merge (dict "podKind" "user") . }}
+  {{- include "jupyterhub.prepareScope" $_ }}
+  {{- if $_.hasTolerations }}
+  singleuser.tolerations-list: |
+    {{- $_.tolerationsList | nindent 4 }}
+  {{- end }}
+  {{- if $_.nodeAffinityRequired }}
+  singleuser.node-affinity-required: |
+    {{- $_.nodeAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.nodeAffinityPreferred }}
+  singleuser.node-affinity-preferred: |
+    {{- $_.nodeAffinityPreferred | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAffinityRequired }}
+  singleuser.pod-affinity-required: |
+    {{- $_.podAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAffinityPreferred }}
+  singleuser.pod-affinity-preferred: |
+    {{- $_.podAffinityPreferred | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAntiAffinityRequired }}
+  singleuser.pod-anti-affinity-required: |
+    {{- $_.podAntiAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAntiAffinityPreferred }}
+  singleuser.pod-anti-affinity-preferred: |
+    {{- $_.podAntiAffinityPreferred | nindent 4 }}
+  {{- end }}
 
   {{- /* KubeSpawner */}}
   kubespawner.common-labels: |

jupyterhub/templates/hub/deployment.yaml

@@ -44,6 +44,14 @@ spec:
                     - key: release
                       operator: In
                       values: [{{ .Release.Name | quote }}]
+      {{- $_ := merge (dict "podKind" "core") . }}
+      {{- $dummy := include "jupyterhub.prepareScope" $_ }}
+      {{- if $_.hasTolerations }}
+      {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
+      {{- end }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       volumes:
         - name: config
           configMap:

jupyterhub/templates/proxy/autohttps/deployment.yaml

@@ -52,6 +52,14 @@ spec:
                     - key: release
                       operator: In
                       values: [{{ .Release.Name | quote }}]
+      {{- $_ := merge (dict "podKind" "core") . }}
+      {{- $dummy := include "jupyterhub.prepareScope" $_ }}
+      {{- if $_.hasTolerations }}
+      {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
+      {{- end }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       containers:
         - name: nginx
           image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}"

jupyterhub/templates/proxy/deployment.yaml

@@ -45,6 +45,14 @@ spec:
                     - key: release
                       operator: In
                       values: [{{ .Release.Name | quote }}]
+      {{- $_ := merge (dict "podKind" "core") . }}
+      {{- $dummy := include "jupyterhub.prepareScope" $_ }}
+      {{- if $_.hasTolerations }}
+      {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
+      {{- end }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       {{- if $manualHTTPS }}
       volumes:
         - name: tls-secret

jupyterhub/templates/scheduling/_scheduling-helpers.tpl

@@ -0,0 +1,191 @@
+{{- /*
+  jupyterhub.prepareScope
+    Requires the .podKind field.
+
+    prepareScope sets fields on the scope to be used by the
+    jupyterhub.tolerations and jupyterhub.affinity helm template functions
+    below.
+*/}}
+{{- define "jupyterhub.prepareScope" -}}
+{{- /* Update the copied scope */}}
+{{- $dummy := set . "component" (include "jupyterhub.componentLabel" .) }}
+
+{{- if .podKind }}
+{{- if eq .podKind "core" -}}
+{{- $dummy := set . "matchNodePurpose" .Values.scheduling.corePods.nodeAffinity.matchNodePurpose }}
+{{- else if eq .podKind "user" -}}
+{{- $dummy := set . "matchNodePurpose" .Values.scheduling.userPods.nodeAffinity.matchNodePurpose }}
+{{- end }}
+{{- end }}
+
+
+{{- /* Fetch relevant information */}}
+{{- $dummy := set . "tolerationsList" (include "jupyterhub.tolerationsList" .) }}
+{{- $dummy := set . "tolerations" (include "jupyterhub.tolerations" .) }}
+{{- $dummy := set . "hasTolerations" .tolerations }}
+
+{{- $dummy := set . "nodeAffinityRequired" (include "jupyterhub.nodeAffinityRequired" .) }}
+{{- $dummy := set . "podAffinityRequired" (include "jupyterhub.podAffinityRequired" .) }}
+{{- $dummy := set . "podAntiAffinityRequired" (include "jupyterhub.podAntiAffinityRequired" .) }}
+{{- $dummy := set . "nodeAffinityPreferred" (include "jupyterhub.nodeAffinityPreferred" .) }}
+{{- $dummy := set . "podAffinityPreferred" (include "jupyterhub.podAffinityPreferred" .) }}
+{{- $dummy := set . "podAntiAffinityPreferred" (include "jupyterhub.podAntiAffinityPreferred" .) }}
+{{- $dummy := set . "hasNodeAffinity" (or .nodeAffinityRequired .nodeAffinityPreferred) }}
+{{- $dummy := set . "hasPodAffinity" (or .podAffinityRequired .podAffinityPreferred) }}
+{{- $dummy := set . "hasPodAntiAffinity" (or .podAntiAffinityRequired .podAntiAffinityPreferred) }}
+{{- $dummy := set . "hasAffinity" (or .hasNodeAffinity .hasPodAffinity .hasPodAntiAffinity) }}
+{{- end }}
+
+
+
+{{- /*
+  jupyterhub.tolerations
+    Refactors the setting of the user pods tolerations field.
+*/}}
+{{- define "jupyterhub.tolerations" -}}
+{{- if .tolerationsList -}}
+tolerations:
+  {{- .tolerationsList | nindent 2 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.tolerationsList" -}}
+{{- if eq .podKind "core" -}}
+{{- else if eq .podKind "user" -}}
+- key: hub.jupyter.org_dedicated
+  operator: Equal
+  value: user
+  effect: NoSchedule
+{{- if .Values.singleuser.extraTolerations -}}
+{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+
+
+{{- define "jupyterhub.nodeAffinityRequired" -}}
+{{- if eq .matchNodePurpose "require" -}}
+- matchExpressions:
+  - key: hub.jupyter.org/node-purpose
+    operator: In
+    values: [{{ .podKind }}]
+{{- end }}
+{{- if .Values.singleuser.extraNodeAffinity.required -}}
+{{- .Values.singleuser.extraNodeAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.nodeAffinityPreferred" -}}
+{{- if eq .matchNodePurpose "prefer" -}}
+- weight: 100
+  preference:
+    matchExpressions:
+      - key: hub.jupyter.org/node-purpose
+        operator: In
+        values: [{{ .podKind }}]
+{{- end }}
+{{- if .Values.singleuser.extraNodeAffinity.preferred -}}
+{{- .Values.singleuser.extraNodeAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.podAffinityRequired" -}}
+{{- if .Values.singleuser.extraPodAffinity.required }}
+{{- .Values.singleuser.extraPodAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.podAffinityPreferred" -}}
+{{- if .Values.scheduling.userPods.podAffinity.preferScheduleNextToRealUsers -}}
+- weight: 100
+  podAffinityTerm:
+    labelSelector:
+      matchExpressions:
+        - key: component
+          operator: In
+          values: [singleuser-server]
+    topologyKey: kubernetes.io/hostname
+{{- end }}
+{{- if .Values.singleuser.extraPodAffinity.preferred -}}
+{{- .Values.singleuser.extraPodAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.podAntiAffinityRequired" -}}
+{{- if .Values.singleuser.extraPodAntiAffinity.required -}}
+{{- .Values.singleuser.extraPodAntiAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+{{- define "jupyterhub.podAntiAffinityPreferred" -}}
+{{- if eq .component "scheduler" -}}
+- weight: 100
+  podAffinityTerm:
+    labelSelector:
+      matchExpressions:
+        - key: component
+          operator: In
+          values: [scheduler]
+    topologyKey: kubernetes.io/hostname
+{{- end }}
+{{- if .Values.singleuser.extraPodAntiAffinity.preferred -}}
+{{- .Values.singleuser.extraPodAntiAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }}
+{{- end }}
+{{- end }}
+
+
+
+{{- /*
+  input: podKind
+*/}}
+{{- define "jupyterhub.affinity" -}}
+{{- /*
+  HACK: Creating a copy of the scope to avoid modification of the scope passed
+  to us. We will reference this copy from within the if statements in order to
+  be able to set a variable outside the if statements local scope.
+*/}}
+{{- /* Render the affinity entry */}}
+{{- if .hasAffinity -}}
+affinity:
+  {{- if .hasNodeAffinity }}
+  nodeAffinity:
+    {{- if .nodeAffinityRequired }}
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        {{- .nodeAffinityRequired | nindent 8 }}
+    {{- end }}
+
+    {{- if .nodeAffinityPreferred }}
+    preferredDuringSchedulingIgnoredDuringExecution:
+      {{- .nodeAffinityPreferred | nindent 6 }}
+    {{- end }}
+  {{- end }}
+
+  {{- if .hasPodAffinity }}
+  podAffinity:
+    {{- if .podAffinityRequired }}
+    requiredDuringSchedulingIgnoredDuringExecution:
+      {{- .podAffinityRequired | nindent 6 }}
+    {{- end }}
+
+    {{- if .podAffinityPreferred }}
+    preferredDuringSchedulingIgnoredDuringExecution:
+      {{- .podAffinityPreferred | nindent 6 }}
+    {{- end }}
+  {{- end }}
+
+  {{- if .hasPodAntiAffinity }}
+  podAntiAffinity:
+    {{- if .podAntiAffinityRequired }}
+    requiredDuringSchedulingIgnoredDuringExecution:
+      {{- .podAntiAffinityRequired | nindent 6 }}
+    {{- end }}
+
+    {{- if .podAntiAffinityPreferred }}
+    preferredDuringSchedulingIgnoredDuringExecution:
+      {{- .podAntiAffinityPreferred | nindent 6 }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}

jupyterhub/values.yaml

@@ -127,6 +127,17 @@ auth:
 
 
 singleuser:
+  nodeSelector: {}
+  extraTolerations: []
+  extraNodeAffinity:
+    required: []
+    preferred: []
+  extraPodAffinity:
+    required: []
+    preferred: []
+  extraPodAntiAffinity:
+    required: []
+    preferred: []
   networkTools:
     image:
       name: jupyterhub/k8s-network-tools

tools/lint-chart-values.yaml

@@ -103,6 +103,17 @@ auth:
 
 
 singleuser:
+  nodeSelector: {}
+  extraTolerations: []
+  extraNodeAffinity:
+    required: []
+    preferred: []
+  extraPodAffinity:
+    required: []
+    preferred: []
+  extraPodAntiAffinity:
+    required: []
+    preferred: []
   networkTools:
     image:
       name: jupyterhub/k8s-network-tools
@@ -123,7 +134,6 @@ singleuser:
   extraEnv: {}
   lifecycleHooks:
   initContainers:
-  nodeSelector: {}
   uid: 1000
   fsGid: 100
   serviceAccountName: