diff --git a/images/hub/jupyterhub_config.py b/images/hub/jupyterhub_config.py index f9cae389c4..68c60eaaff 100644 --- a/images/hub/jupyterhub_config.py +++ b/images/hub/jupyterhub_config.py @@ -62,11 +62,19 @@ ('fs_gid', 'fs-gid'), ('service_account', 'service-account-name'), ('scheduler_name', 'scheduler-name'), - ('node_selector', 'node-selector'), ): set_config_if_not_none(c.KubeSpawner, trait, 'singleuser.' + cfg_key) +c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {}) c.KubeSpawner.image_spec = get_config('singleuser.image-spec') +c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations', [])) +c.KubeSpawner.node_selector.update(get_config('singleuser.node-selector', {})) +c.KubeSpawner.node_affinity_required.extend(get_config('singleuser.node-affinity-required', [])) +c.KubeSpawner.node_affinity_preferred.extend(get_config('singleuser.node-affinity-preferred', [])) +c.KubeSpawner.pod_affinity_required.extend(get_config('singleuser.pod-affinity-required', [])) +c.KubeSpawner.pod_affinity_preferred.extend(get_config('singleuser.pod-affinity-preferred', [])) +c.KubeSpawner.pod_anti_affinity_required.extend(get_config('singleuser.pod-anti-affinity-required', [])) +c.KubeSpawner.pod_anti_affinity_preferred.extend(get_config('singleuser.pod-anti-affinity-preferred', [])) # Configure dynamically provisioning pvc storage_type = get_config('singleuser.storage.type') if storage_type == 'dynamic': diff --git a/jupyterhub/schema.yaml b/jupyterhub/schema.yaml index dd7baacd04..e41b4ee69a 100644 --- a/jupyterhub/schema.yaml +++ b/jupyterhub/schema.yaml @@ -460,6 +460,81 @@ properties: description: | Deprecated and no longer does anything. Use the user-scheduler instead in order to accomplish a good packing of the user pods. + extraTolerations: + type: list + description: | + Tolerations allow a pod to be scheduled on nodes with taints. These + are additional tolerations other than the user pods and core pods + default ones `hub.jupyter.org/dedicated=user:NoSchedule` or + `hub.jupyter.org/dedicated=core:NoSchedule`. Note that a duplicate set + of tolerations exist where `/` is replaced with `_` as the Google + cloud does not support the character `/` yet in the toleration. + + See the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + for more info. + + Pass this field an array of + [`Toleration`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core) + objects. + extraNodeAffinity: + type: object + description: | + Affinities describe where pods prefer or require to be scheduled, they + may prefer or require a node where they are to be scheduled to have a + certain label (node affinity). They may also require to be scheduled + in proximity or with a lack of proximity to another pod (pod affinity + and anti pod affinity). + + See the [Kubernetes + docs](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) + for more info. + properties: + required: + type: list + description: | + Pass this field an array of + [`NodeSelectorTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#nodeselectorterm-v1-core) + objects. + preferred: + type: list + description: | + Pass this field an array of + [`PreferredSchedulingTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#preferredschedulingterm-v1-core) + objects. + extraPodAffinity: + type: object + description: | + See the description of `singleuser.extraNodeAffinity`. + properties: + required: + type: list + description: | + Pass this field an array of + [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core) + objects. + preferred: + type: list + description: | + Pass this field an array of + [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core) + objects. + extraPodAntiAffinity: + type: object + description: | + See the description of `singleuser.extraNodeAffinity`. + properties: + required: + type: list + description: | + Pass this field an array of + [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core) + objects. + preferred: + type: list + description: | + Pass this field an array of + [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core) + objects. scheduling: @@ -495,3 +570,51 @@ properties: type: - string - "null" + corePods: + type: object + description: | + These settings influence the core pods like the hub, proxy and + user-scheduler pods. + properties: + nodeAffinity: + type: object + description: | + Where should pods be scheduled? Perhaps on nodes with a certain + label is preferred or even required? + properties: + matchNodePurpose: + type: string + enum: + - ignore + - prefer + - require + description: | + Decide if core pods *ignore*, *prefer* or *require* to + schedule on nodes with this label: + ``` + hub.jupyter.org/node-purpose=core + ``` + userPods: + type: object + description: | + These settings influence the user pods like the user-placeholder, + user-dummy and actual user pods named like jupyter-someusername. + properties: + nodeAffinity: + type: object + description: | + Where should pods be scheduled? Perhaps on nodes with a certain + label is preferred or even required? + properties: + matchNodePurpose: + type: string + enum: + - ignore + - prefer + - require + description: | + Decide if user pods *ignore*, *prefer* or *require* to + schedule on nodes with this label: + ``` + hub.jupyter.org/node-purpose=user + ``` diff --git a/jupyterhub/templates/hub/configmap.yaml b/jupyterhub/templates/hub/configmap.yaml index 3f8bd1eaa2..2ea7f937b6 100644 --- a/jupyterhub/templates/hub/configmap.yaml +++ b/jupyterhub/templates/hub/configmap.yaml @@ -208,6 +208,12 @@ data: {{- range $key, $value := .Values.singleuser.extraLabels }} {{ $key | quote }}: {{ $value | quote }} {{- end }} + {{- if .Values.singleuser.storage.extraLabels }} + singleuser.storage-extra-labels: | + {{- range $key, $value := .Values.singleuser.storage.extraLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- end }} {{- if .Values.singleuser.extraEnv }} singleuser.extra-env: | {{- range $key, $value := .Values.singleuser.extraEnv }} @@ -215,6 +221,34 @@ data: {{- end }} {{- end }} + singleuser.tolerations: | + {{- include "jupyterhub.userTolerations" . | nindent 4 }} + + {{- if include "jupyterhub.userNodeAffinityRequired" . }} + singleuser.node-affinity-required: | + {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 4 }} + {{- end }} + {{- if include "jupyterhub.userNodeAffinityPreferred" . }} + singleuser.node-affinity-preferred: | + {{- include "jupyterhub.userNodeAffinityPreferred" . | nindent 4 }} + {{- end }} + {{- if include "jupyterhub.userPodAffinityRequired" . }} + singleuser.pod-affinity-required: | + {{- include "jupyterhub.userPodAffinityRequired" . | nindent 4 }} + {{- end }} + {{- if include "jupyterhub.userPodAffinityPreferred" . }} + singleuser.pod-affinity-preferred: | + {{- include "jupyterhub.userPodAffinityPreferred" . | nindent 4 }} + {{- end }} + {{- if include "jupyterhub.userPodAntiAffinityRequired" . }} + singleuser.pod-anti-affinity-required: | + {{- include "jupyterhub.userPodAntiAffinityRequired" . | nindent 4 }} + {{- end }} + {{- if include "jupyterhub.userPodAntiAffinityPreferred" . }} + singleuser.pod-anti-affinity-preferred: | + {{- include "jupyterhub.userPodAntiAffinityPreferred" . | nindent 4 }} + {{- end }} + {{- if .Values.scheduling.userScheduler.enabled }} singleuser.scheduler-name: "{{ .Release.Name }}-user-scheduler" {{- end }} diff --git a/jupyterhub/templates/hub/deployment.yaml b/jupyterhub/templates/hub/deployment.yaml index 028363162b..82cb2ced65 100644 --- a/jupyterhub/templates/hub/deployment.yaml +++ b/jupyterhub/templates/hub/deployment.yaml @@ -31,20 +31,7 @@ spec: {{- end }} spec: nodeSelector: {{ toJson .Values.hub.nodeSelector }} - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: component - operator: In - values: ['proxy'] - - key: release - operator: In - values: [{{ .Release.Name | quote }}] + {{- include "jupyterhub.coreAffinity" . | nindent 6 }} volumes: - name: config configMap: diff --git a/jupyterhub/templates/image-puller/_daemonset-helper.yaml b/jupyterhub/templates/image-puller/_daemonset-helper.yaml index ed0ec4e522..e362325a9e 100644 --- a/jupyterhub/templates/image-puller/_daemonset-helper.yaml +++ b/jupyterhub/templates/image-puller/_daemonset-helper.yaml @@ -36,6 +36,16 @@ spec: {{- /* Changes here will cause the DaemonSet to restart the pods. */}} {{- include "jupyterhub.matchLabels" . | nindent 8 }} spec: + tolerations: + {{- include "jupyterhub.userTolerations" . | nindent 8 }} + nodeSelector: {{ toJson .Values.singleuser.nodeSelector }} + {{- if include "jupyterhub.userNodeAffinityRequired" . }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 14 }} + {{- end }} terminationGracePeriodSeconds: 0 automountServiceAccountToken: false {{- if .Values.singleuser.imagePullSecret.enabled }} @@ -77,7 +87,6 @@ spec: - -c - echo "Pulling complete" {{- end }} - nodeSelector: {{ toJson .Values.singleuser.nodeSelector }} containers: - name: pause image: {{ .Values.prePuller.pause.image.name }}:{{ .Values.prePuller.pause.image.tag }} diff --git a/jupyterhub/templates/proxy/autohttps/deployment.yaml b/jupyterhub/templates/proxy/autohttps/deployment.yaml index 907be7553f..8409b5c95e 100644 --- a/jupyterhub/templates/proxy/autohttps/deployment.yaml +++ b/jupyterhub/templates/proxy/autohttps/deployment.yaml @@ -35,22 +35,9 @@ spec: {{- if .Values.rbac.enabled }} serviceAccountName: autohttps {{- end }} - nodeSelector: {{ toJson .Values.proxy.nodeSelector }} terminationGracePeriodSeconds: 60 - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: component - operator: In - values: ['hub'] - - key: release - operator: In - values: [{{ .Release.Name | quote }}] + nodeSelector: {{ toJson .Values.proxy.nodeSelector }} + {{- include "jupyterhub.coreAffinity" . | nindent 6 }} containers: - name: nginx image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}" diff --git a/jupyterhub/templates/proxy/deployment.yaml b/jupyterhub/templates/proxy/deployment.yaml index 4d022213e3..df2e54ad56 100644 --- a/jupyterhub/templates/proxy/deployment.yaml +++ b/jupyterhub/templates/proxy/deployment.yaml @@ -29,22 +29,9 @@ spec: {{- .Values.proxy.annotations | toYaml | trimSuffix "\n" | nindent 8 }} {{- end }} spec: - nodeSelector: {{ toJson .Values.proxy.nodeSelector }} terminationGracePeriodSeconds: 60 - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: component - operator: In - values: ['hub'] - - key: release - operator: In - values: [{{ .Release.Name | quote }}] + nodeSelector: {{ toJson .Values.proxy.nodeSelector }} + {{- include "jupyterhub.coreAffinity" . | nindent 6 }} {{- if $manualHTTPSwithsecret }} volumes: - name: tls-secret diff --git a/jupyterhub/templates/scheduling/_scheduling-helpers.tpl b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl new file mode 100644 index 0000000000..c1c19cb849 --- /dev/null +++ b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl @@ -0,0 +1,99 @@ +{{- /* + jupyterhub.userTolerations + Lists the tolerations for node taints that the user pods should have +*/}} +{{- define "jupyterhub.userTolerations" -}} +- key: hub.jupyter.org_dedicated + operator: Equal + value: user + effect: NoSchedule +- key: hub.jupyter.org/dedicated + operator: Equal + value: user + effect: NoSchedule +{{- if .Values.singleuser.extraTolerations }} +{{- .Values.singleuser.extraTolerations | toYaml | trimSuffix "\n" | nindent 0 }} +{{- end }} +{{- end }} + + + +{{- define "jupyterhub.userNodeAffinityRequired" -}} +{{- if eq .Values.scheduling.userPods.nodeAffinity.matchNodePurpose "require" -}} +- matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [user] + {{- if .Values.singleuser.extraNodeAffinity.required }}{{ println }}{{ end }} +{{- end }} +{{- if .Values.singleuser.extraNodeAffinity.required -}} +{{ .Values.singleuser.extraNodeAffinity.required | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.userNodeAffinityPreferred" -}} +{{- if eq .Values.scheduling.userPods.nodeAffinity.matchNodePurpose "prefer" -}} +- weight: 100 + preference: + matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [user] + {{- if .Values.singleuser.extraNodeAffinity.preferred }}{{ println }}{{ end }} +{{- end }} +{{- if .Values.singleuser.extraNodeAffinity.preferred -}} +{{ .Values.singleuser.extraNodeAffinity.preferred | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.userPodAffinityRequired" -}} +{{- if .Values.singleuser.extraPodAffinity.required -}} +{{ .Values.singleuser.extraPodAffinity.required | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.userPodAffinityPreferred" -}} +{{- if .Values.singleuser.extraPodAffinity.preferred -}} +{{ .Values.singleuser.extraPodAffinity.preferred | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.userPodAntiAffinityRequired" -}} +{{- if .Values.singleuser.extraPodAntiAffinity.required -}} +{{ .Values.singleuser.extraPodAntiAffinity.required | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + +{{- define "jupyterhub.userPodAntiAffinityPreferred" -}} +{{- if .Values.singleuser.extraPodAntiAffinity.preferred -}} +{{ .Values.singleuser.extraPodAntiAffinity.preferred | toYaml | trimSuffix "\n" }} +{{- end }} +{{- end }} + + + +{{- define "jupyterhub.coreAffinity" -}} +{{- $require := eq .Values.scheduling.corePods.nodeAffinity.matchNodePurpose "require" -}} +{{- $prefer := eq .Values.scheduling.corePods.nodeAffinity.matchNodePurpose "prefer" -}} +{{- if or $require $prefer -}} +affinity: + nodeAffinity: + {{- if $require }} + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [core] + {{- end }} + {{- if $prefer }} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: [core] + {{- end }} +{{- end }} +{{- end }} diff --git a/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml b/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml index 539e884947..9f462481e0 100644 --- a/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml +++ b/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml @@ -24,6 +24,7 @@ spec: serviceAccountName: user-scheduler {{- end }} nodeSelector: {{ toJson .Values.scheduling.userScheduler.nodeSelector }} + {{- include "jupyterhub.coreAffinity" . | nindent 6 }} containers: - name: user-scheduler image: {{ include "jupyterhub.scheduler.image" . }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index eec2bfac2b..7742ab5c2d 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -142,6 +142,17 @@ auth: singleuser: + extraTolerations: [] + nodeSelector: {} + extraNodeAffinity: + required: [] + preferred: [] + extraPodAffinity: + required: [] + preferred: [] + extraPodAntiAffinity: + required: [] + preferred: [] networkTools: image: name: jupyterhub/k8s-network-tools @@ -165,12 +176,12 @@ singleuser: lifecycleHooks: initContainers: [] extraContainers: [] - nodeSelector: {} uid: 1000 fsGid: 100 serviceAccountName: storage: type: dynamic + extraLabels: {} extraVolumes: [] extraVolumeMounts: [] static: @@ -223,6 +234,12 @@ scheduling: requests: cpu: 50m memory: 256Mi + corePods: + nodeAffinity: + matchNodePurpose: prefer + userPods: + nodeAffinity: + matchNodePurpose: prefer prePuller: diff --git a/tools/templates/lint-and-validate-values.yaml b/tools/templates/lint-and-validate-values.yaml index 8552b90502..d6d6b99d33 100644 --- a/tools/templates/lint-and-validate-values.yaml +++ b/tools/templates/lint-and-validate-values.yaml @@ -105,7 +105,7 @@ proxy: enabled: true https: enabled: true - type: manual + type: letsencrypt #type: letsencrypt, manual, secret letsencrypt: contactEmail: 'e@domain.com' @@ -149,16 +149,58 @@ auth: singleuser: nodeSelector: mock-node-selector: mock - extraTolerations: [] + extraTolerations: + - key: hub.jupyter.org/test + operator: Equal + value: test + effect: NoSchedule extraNodeAffinity: - required: [] - preferred: [] + required: + - matchExpressions: + - key: hub.jupyter.org/test-required-node + operator: In + values: [test] + preferred: + - weight: 10 + preference: + matchExpressions: + - key: hub.jupyter.org/test-preferred-node + operator: In + values: [test] extraPodAffinity: - required: [] - preferred: [] + required: + - labelSelector: + matchExpressions: + - key: hub.jupyter.org/test-required-pod + operator: In + values: [test] + topologyKey: failure-domain.beta.kubernetes.io/zone + preferred: + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: hub.jupyter.org/test-preferred-pod + operator: In + values: [test] + topologyKey: kubernetes.io/hostname extraPodAntiAffinity: - required: [] - preferred: [] + required: + - labelSelector: + matchExpressions: + - key: hub.jupyter.org/test-required-anti-pod + operator: In + values: [test] + topologyKey: failure-domain.beta.kubernetes.io/zone + preferred: + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: hub.jupyter.org/test-preferred-anti-pod + operator: In + values: [test] + topologyKey: kubernetes.io/hostname cloudMetadata: enabled: true ip: 169.254.169.254 @@ -172,7 +214,6 @@ singleuser: - 169.254.169.254/32 events: true extraLabels: {} - storageExtraLabels: {} extraEnv: {} lifecycleHooks: initContainers: @@ -188,6 +229,8 @@ singleuser: serviceAccountName: storage: type: dynamic + extraLabels: + mock-label: mock-value extraVolumes: [] extraVolumeMounts: [] static: @@ -229,6 +272,12 @@ scheduling: image: name: gcr.io/google_containers/kube-scheduler-amd64 tag: v1.11.2 + corePods: + nodeAffinity: + matchNodePurpose: require + userPods: + nodeAffinity: + matchNodePurpose: require prePuller: