diff --git a/images/hub/jupyterhub_config.py b/images/hub/jupyterhub_config.py index cda12cffe9..b6cc6674fb 100644 --- a/images/hub/jupyterhub_config.py +++ b/images/hub/jupyterhub_config.py @@ -65,6 +65,14 @@ c.KubeSpawner.service_account = service_account_name c.KubeSpawner.node_selector = get_config('singleuser.node-selector') + +c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations-list', [])) +c.KubeSpawner.node_affinity_required.extend(get_config('singleuser.node-affinity-required', [])) +c.KubeSpawner.node_affinity_preferred.extend(get_config('singleuser.node-affinity-preferred', [])) +c.KubeSpawner.pod_affinity_required.extend(get_config('singleuser.pod-affinity-required', [])) +c.KubeSpawner.pod_affinity_preferred.extend(get_config('singleuser.pod-affinity-preferred', [])) +c.KubeSpawner.pod_anti_affinity_required.extend(get_config('singleuser.pod-anti-affinity-required', [])) +c.KubeSpawner.pod_anti_affinity_preferred.extend(get_config('singleuser.pod-anti-affinity-preferred', [])) # Configure dynamically provisioning pvc storage_type = get_config('singleuser.storage.type') if storage_type == 'dynamic': diff --git a/jupyterhub/templates/hub/configmap.yaml b/jupyterhub/templates/hub/configmap.yaml index 03f6cf3a95..625e40e043 100644 --- a/jupyterhub/templates/hub/configmap.yaml +++ b/jupyterhub/templates/hub/configmap.yaml @@ -187,7 +187,37 @@ data: {{ $key | quote }}: {{ $value | quote }} {{- end }} {{- end }} - + + {{- $_ := merge (dict "podKind" "user") . }} + {{- include "jupyterhub.prepareScope" $_ }} + {{- if $_.hasTolerations }} + singleuser.tolerations-list: | + {{- $_.tolerationsList | nindent 4 }} + {{- end }} + {{- if $_.nodeAffinityRequired }} + singleuser.node-affinity-required: | + {{- $_.nodeAffinityRequired | nindent 4 }} + {{- end }} + {{- if $_.nodeAffinityPreferred }} + singleuser.node-affinity-preferred: | + {{- $_.nodeAffinityPreferred | nindent 4 }} + {{- end }} + {{- if $_.podAffinityRequired }} + singleuser.pod-affinity-required: | + {{- $_.podAffinityRequired | nindent 4 }} + {{- end }} + {{- if $_.podAffinityPreferred }} + singleuser.pod-affinity-preferred: | + {{- $_.podAffinityPreferred | nindent 4 }} + {{- end }} + {{- if $_.podAntiAffinityRequired }} + singleuser.pod-anti-affinity-required: | + {{- $_.podAntiAffinityRequired | nindent 4 }} + {{- end }} + {{- if $_.podAntiAffinityPreferred }} + singleuser.pod-anti-affinity-preferred: | + {{- $_.podAntiAffinityPreferred | nindent 4 }} + {{- end }} {{- /* KubeSpawner */}} kubespawner.common-labels: | diff --git a/jupyterhub/templates/hub/deployment.yaml b/jupyterhub/templates/hub/deployment.yaml index dd50445721..b1992946b7 100644 --- a/jupyterhub/templates/hub/deployment.yaml +++ b/jupyterhub/templates/hub/deployment.yaml @@ -44,6 +44,14 @@ spec: - key: release operator: In values: [{{ .Release.Name | quote }}] + {{- $_ := merge (dict "podKind" "core") . }} + {{- $dummy := include "jupyterhub.prepareScope" $_ }} + {{- if $_.hasTolerations }} + {{- include "jupyterhub.tolerations" $_ | nindent 6 }} + {{- end }} + {{- if $_.hasAffinity }} + {{- include "jupyterhub.affinity" $_ | nindent 6 }} + {{- end }} volumes: - name: config configMap: diff --git a/jupyterhub/templates/proxy/autohttps/deployment.yaml b/jupyterhub/templates/proxy/autohttps/deployment.yaml index cd203daa69..42d43e58d2 100644 --- a/jupyterhub/templates/proxy/autohttps/deployment.yaml +++ b/jupyterhub/templates/proxy/autohttps/deployment.yaml @@ -52,6 +52,14 @@ spec: - key: release operator: In values: [{{ .Release.Name | quote }}] + {{- $_ := merge (dict "podKind" "core") . }} + {{- $dummy := include "jupyterhub.prepareScope" $_ }} + {{- if $_.hasTolerations }} + {{- include "jupyterhub.tolerations" $_ | nindent 6 }} + {{- end }} + {{- if $_.hasAffinity }} + {{- include "jupyterhub.affinity" $_ | nindent 6 }} + {{- end }} containers: - name: nginx image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}" diff --git a/jupyterhub/templates/proxy/deployment.yaml b/jupyterhub/templates/proxy/deployment.yaml index f81e4e2794..6b3de38c86 100644 --- a/jupyterhub/templates/proxy/deployment.yaml +++ b/jupyterhub/templates/proxy/deployment.yaml @@ -45,6 +45,14 @@ spec: - key: release operator: In values: [{{ .Release.Name | quote }}] + {{- $_ := merge (dict "podKind" "core") . }} + {{- $dummy := include "jupyterhub.prepareScope" $_ }} + {{- if $_.hasTolerations }} + {{- include "jupyterhub.tolerations" $_ | nindent 6 }} + {{- end }} + {{- if $_.hasAffinity }} + {{- include "jupyterhub.affinity" $_ | nindent 6 }} + {{- end }} {{- if $manualHTTPS }} volumes: - name: tls-secret diff --git a/jupyterhub/templates/scheduling/_scheduling-helpers.tpl b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl new file mode 100644 index 0000000000..de0894936b --- /dev/null +++ b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl @@ -0,0 +1,191 @@ +{{- /* + jupyterhub.prepareScope + Requires the .podKind field. + + prepareScope sets fields on the scope to be used by the + jupyterhub.tolerations and jupyterhub.affinity helm template functions + below. +*/}} +{{- define "jupyterhub.prepareScope" -}} +{{- /* Update the copied scope */}} +{{- $dummy := set . "component" (include "jupyterhub.componentLabel" .) }} + +{{- if .podKind }} +{{- if eq .podKind "core" -}} +{{- $dummy := set . "matchNodePurpose" .Values.scheduling.corePods.nodeAffinity.matchNodePurpose }} +{{- else if eq .podKind "user" -}} +{{- $dummy := set . "matchNodePurpose" .Values.scheduling.userPods.nodeAffinity.matchNodePurpose }} +{{- end }} +{{- end }} + + +{{- /* Fetch relevant information */}} +{{- $dummy := set . "tolerationsList" (include "jupyterhub.tolerationsList" .) }} +{{- $dummy := set . "tolerations" (include "jupyterhub.tolerations" .) }} +{{- $dummy := set . "hasTolerations" .tolerations }} + +{{- $dummy := set . "nodeAffinityRequired" (include "jupyterhub.nodeAffinityRequired" .) }} +{{- $dummy := set . "podAffinityRequired" (include "jupyterhub.podAffinityRequired" .) }} +{{- $dummy := set . "podAntiAffinityRequired" (include "jupyterhub.podAntiAffinityRequired" .) }} +{{- $dummy := set . "nodeAffinityPreferred" (include "jupyterhub.nodeAffinityPreferred" .) }} +{{- $dummy := set . "podAffinityPreferred" (include "jupyterhub.podAffinityPreferred" .) }} +{{- $dummy := set . "podAntiAffinityPreferred" (include "jupyterhub.podAntiAffinityPreferred" .) }} +{{- $dummy := set . "hasNodeAffinity" (or .nodeAffinityRequired .nodeAffinityPreferred) }} +{{- $dummy := set . "hasPodAffinity" (or .podAffinityRequired .podAffinityPreferred) }} +{{- $dummy := set . "hasPodAntiAffinity" (or .podAntiAffinityRequired .podAntiAffinityPreferred) }} +{{- $dummy := set . "hasAffinity" (or .hasNodeAffinity .hasPodAffinity .hasPodAntiAffinity) }} +{{- end }} + + + +{{- /* + jupyterhub.tolerations + Refactors the setting of the user pods tolerations field. +*/}} +{{- define "jupyterhub.tolerations" -}} +{{- if .tolerationsList -}} +tolerations: + {{- .tolerationsList | nindent 2 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.tolerationsList" -}} +{{- if eq .podKind "core" -}} +{{- else if eq .podKind "user" -}} +- key: hub.jupyter.org_dedicated + operator: Equal + value: user + effect: NoSchedule +{{- if .Values.singleuser.extraTolerations -}} +{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} +{{- end }} + + + +{{- define "jupyterhub.nodeAffinityRequired" -}} +{{- if eq .matchNodePurpose "require" -}} +- matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [{{ .podKind }}] +{{- end }} +{{- if .Values.singleuser.extraNodeAffinity.required -}} +{{- .Values.singleuser.extraNodeAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.nodeAffinityPreferred" -}} +{{- if eq .matchNodePurpose "prefer" -}} +- weight: 100 + preference: + matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [{{ .podKind }}] +{{- end }} +{{- if .Values.singleuser.extraNodeAffinity.preferred -}} +{{- .Values.singleuser.extraNodeAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.podAffinityRequired" -}} +{{- if .Values.singleuser.extraPodAffinity.required }} +{{- .Values.singleuser.extraPodAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.podAffinityPreferred" -}} +{{- if .Values.scheduling.userPods.podAffinity.preferScheduleNextToRealUsers -}} +- weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: component + operator: In + values: [singleuser-server] + topologyKey: kubernetes.io/hostname +{{- end }} +{{- if .Values.singleuser.extraPodAffinity.preferred -}} +{{- .Values.singleuser.extraPodAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.podAntiAffinityRequired" -}} +{{- if .Values.singleuser.extraPodAntiAffinity.required -}} +{{- .Values.singleuser.extraPodAntiAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.podAntiAffinityPreferred" -}} +{{- if eq .component "scheduler" -}} +- weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: component + operator: In + values: [scheduler] + topologyKey: kubernetes.io/hostname +{{- end }} +{{- if .Values.singleuser.extraPodAntiAffinity.preferred -}} +{{- .Values.singleuser.extraPodAntiAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + + + +{{- /* + input: podKind +*/}} +{{- define "jupyterhub.affinity" -}} +{{- /* + HACK: Creating a copy of the scope to avoid modification of the scope passed + to us. We will reference this copy from within the if statements in order to + be able to set a variable outside the if statements local scope. +*/}} +{{- /* Render the affinity entry */}} +{{- if .hasAffinity -}} +affinity: + {{- if .hasNodeAffinity }} + nodeAffinity: + {{- if .nodeAffinityRequired }} + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + {{- .nodeAffinityRequired | nindent 8 }} + {{- end }} + + {{- if .nodeAffinityPreferred }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- .nodeAffinityPreferred | nindent 6 }} + {{- end }} + {{- end }} + + {{- if .hasPodAffinity }} + podAffinity: + {{- if .podAffinityRequired }} + requiredDuringSchedulingIgnoredDuringExecution: + {{- .podAffinityRequired | nindent 6 }} + {{- end }} + + {{- if .podAffinityPreferred }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- .podAffinityPreferred | nindent 6 }} + {{- end }} + {{- end }} + + {{- if .hasPodAntiAffinity }} + podAntiAffinity: + {{- if .podAntiAffinityRequired }} + requiredDuringSchedulingIgnoredDuringExecution: + {{- .podAntiAffinityRequired | nindent 6 }} + {{- end }} + + {{- if .podAntiAffinityPreferred }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- .podAntiAffinityPreferred | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index de274ed3fb..ce38c98f81 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -127,6 +127,17 @@ auth: singleuser: + nodeSelector: {} + extraTolerations: [] + extraNodeAffinity: + required: [] + preferred: [] + extraPodAffinity: + required: [] + preferred: [] + extraPodAntiAffinity: + required: [] + preferred: [] networkTools: image: name: jupyterhub/k8s-network-tools diff --git a/tools/lint-chart-values.yaml b/tools/lint-chart-values.yaml index 7e44c0225a..6cd69a771e 100644 --- a/tools/lint-chart-values.yaml +++ b/tools/lint-chart-values.yaml @@ -103,6 +103,17 @@ auth: singleuser: + nodeSelector: {} + extraTolerations: [] + extraNodeAffinity: + required: [] + preferred: [] + extraPodAffinity: + required: [] + preferred: [] + extraPodAntiAffinity: + required: [] + preferred: [] networkTools: image: name: jupyterhub/k8s-network-tools @@ -123,7 +134,6 @@ singleuser: extraEnv: {} lifecycleHooks: initContainers: - nodeSelector: {} uid: 1000 fsGid: 100 serviceAccountName: