[Part 12 of #758] Preset: scheduling.podAffinity.preferScheduleNextToRealUsers

images/hub/jupyterhub_config.py

@@ -45,29 +45,44 @@
 
 # the hub should listen on all interfaces, so the proxy can access it
 c.JupyterHub.hub_ip = '0.0.0.0'
+c.KubeSpawner.extra_labels = get_config('singleuser.extra-labels', {})
+c.KubeSpawner.extra_labels.update({
+    "hub.jupyter.org/pod-kind": "user"
+})
 
 set_config_if_not_none(c.KubeSpawner, 'common_labels', 'kubespawner.common-labels')
 
 c.KubeSpawner.namespace = os.environ.get('POD_NAMESPACE', 'default')
 
-# Use env var for this, since we want hub to restart when this changes
-c.KubeSpawner.image_spec = os.environ['SINGLEUSER_IMAGE']
 
 for trait, cfg_key in (
     ('start_timeout', 'start-timeout'),
     ('image_pull_policy', 'image-pull-policy'),
     ('image_pull_secrets', 'image-pull-secret-name'),
     ('events_enabled', 'events'),
-    ('extra_labels', 'extra-labels'),
     ('extra_annotations', 'extra-annotations'),
     ('uid', 'uid'),
     ('fs_gid', 'fs-gid'),
     ('service_account', 'service-account-name'),
     ('scheduler_name', 'scheduler-name'),
-    ('node_selector', 'node-selector'),
 ):
     set_config_if_not_none(c.KubeSpawner, trait, 'singleuser.' + cfg_key)
-
+c.KubeSpawner.storage_extra_labels = get_config('singleuser.storage-extra-labels', {})
+c.KubeSpawner.storage_extra_labels.update({
+    "hub.jupyter.org/storage-kind": "user"
+})
+
+c.KubeSpawner.image_spec = get_config('singleuser.image-spec')
+c.KubeSpawner.priority_class_name = get_config('singleuser.priority-class-name', "")
+
+c.KubeSpawner.tolerations.extend(get_config('singleuser.tolerations', []))
+c.KubeSpawner.node_selector.update(get_config('singleuser.node-selector', {}))
+c.KubeSpawner.node_affinity_required.extend(get_config('singleuser.node-affinity-required', []))
+c.KubeSpawner.node_affinity_preferred.extend(get_config('singleuser.node-affinity-preferred', []))
+c.KubeSpawner.pod_affinity_required.extend(get_config('singleuser.pod-affinity-required', []))
+c.KubeSpawner.pod_affinity_preferred.extend(get_config('singleuser.pod-affinity-preferred', []))
+c.KubeSpawner.pod_anti_affinity_required.extend(get_config('singleuser.pod-anti-affinity-required', []))
+c.KubeSpawner.pod_anti_affinity_preferred.extend(get_config('singleuser.pod-anti-affinity-preferred', []))
 # Configure dynamically provisioning pvc
 storage_type = get_config('singleuser.storage.type')
 if storage_type == 'dynamic':

jupyterhub/schema.yaml

@@ -460,6 +460,81 @@ properties:
         description: |
           Deprecated and no longer does anything. Use the user-scheduler instead
           in order to accomplish a good packing of the user pods.
+      tolerations:
+        type: list
+        description: |
+          Tolerations allow a pod to be scheduled on nodes with taints. These
+          are additional tolerations other than the user pods and core pods
+          default ones `hub.jupyter.org/dedicated=user:NoSchedule` or
+          `hub.jupyter.org/dedicated=core:NoSchedule`. Note that a duplicate set
+          of tolerations exist where `/` is replaced with `_` as the Google
+          cloud does not support the character `/` yet in the toleration.
+
+          See the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)
+          for more info.
+
+          Pass this field an array of
+          [`Toleration`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core)
+          objects.
+      extraNodeAffinity:
+        type: object
+        description: |
+          Affinities describe where pods prefer or require to be scheduled, they
+          may prefer or require a node where they are to be scheduled to have a
+          certain label (node affinity). They may also require to be scheduled
+          in proximity or with a lack of proximity to another pod (pod affinity
+          and anti pod affinity).
+
+          See the [Kubernetes
+          docs](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)
+          for more info.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`NodeSelectorTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#nodeselectorterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PreferredSchedulingTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#preferredschedulingterm-v1-core)
+              objects.
+      extraPodAffinity:
+        type: object
+        description: |
+          See the description of `singleuser.extraNodeAffinity`.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core)
+              objects.
+      extraPodAntiAffinity:
+        type: object
+        description: |
+          See the description of `singleuser.extraNodeAffinity`.
+        properties:
+          required:
+            type: list
+            description: |
+              Pass this field an array of
+              [`PodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podaffinityterm-v1-core)
+              objects.
+          preferred:
+            type: list
+            description: |
+              Pass this field an array of
+              [`WeightedPodAffinityTerm`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#weightedpodaffinityterm-v1-core)
+              objects.
 
 
   scheduling:
@@ -495,3 +570,111 @@ properties:
                 type:
                   - string
                   - "null"
+      podPriority:
+        type: object
+        description: |
+          Generally available since Kubernetes 1.11, Pod Priority is used to
+          allow real users evict placeholder pods.
+        properties:
+          enabled:
+            type: bool
+            description: |
+              Generally available since Kubernetes 1.11, Pod Priority is used to
+              allow real users evict placeholder pods.
+      userPlaceholder:
+        type: object
+        description: |
+          User placeholders simulate users but will thanks to PodPriority be
+          evicted by the cluster autoscaler if a real user shows up. In this way
+          placeholders allow you to create a headroom for the real users and
+          reduce the risk of a user having to wait for a node to be added. Be
+          sure to use the the continuous image puller as well along with
+          placeholders, so the images are also available when real users arrive.
+
+          To test your setup efficiently, you can adjust the amount of user
+          placeholders with the following command:
+          ```sh
+          # Configure to have 3 user placeholders
+          kubectl scale sts/user-placeholder --replicas=3
+          ```
+        properties:
+          enabled:
+            type: bool
+          replicas:
+            type: int
+            description: |
+              How many placeholder pods would you like to have?
+          resources:
+            type: object
+            description: |
+              Unless specified here, the placeholder pods will request the same
+              resources specified for the real singleuser pods.
+      corePods:
+        type: object
+        description: |
+          These settings influence the core pods like the hub, proxy and
+          user-scheduler pods.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if core pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=core
+                  ```
+      userPods:
+        type: object
+        description: |
+          These settings influence the user pods like the user-placeholder,
+          user-dummy and actual user pods named like jupyter-someusername.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if user pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=user
+                  ```
+          podAffinity:
+            type: object
+            description: |
+              When a pod is to be scheduled, the scheduled can be made to prefer
+              of require having other pods within the same domain (commonly the
+              same
+              node).
+            properties:
+              preferScheduleNextToRealUsers:
+                type: bool
+                description: |
+                  Consider two nodes, one with two user-placeholder pods and one
+                  with two actual users. With this preconfigured affinity option
+                  set, the pod will schedule next to the real users which is
+                  what you would want in order to accomplish efficient cluster
+                  autoscaling.
+
+                  NOTE: This is recommended for clusters with less than 100
+                  nodes, but since pod affinities are computationally expensive
+                  and cause the scheduler to leave new pods in Pending state for
+                  seconds upwards if the cluster grows large.

jupyterhub/templates/_helpers.tpl

@@ -192,3 +192,62 @@ component: {{ include "jupyterhub.componentLabel" . }}
 }
 {{- end }}
 {{- end }}
+
+
+{{- /*
+The default resource request - whatever a singleuser requests.
+*/}}
+{{- define "jupyterhub.default-resources" -}}
+{{- $r1 := .Values.singleuser.cpu.guarantee -}}
+{{- $r2 := .Values.singleuser.memory.guarantee -}}
+{{- $r3 := .Values.singleuser.extraResource.guarantees -}}
+{{- $r := or $r1 $r2 $r3 }}
+{{- $l1 := .Values.singleuser.cpu.limit -}}
+{{- $l2 := .Values.singleuser.memory.limit -}}
+{{- $l3 := .Values.singleuser.extraResource.limits -}}
+{{- $l := or $l1 $l2 $l3 }}
+{{- if $r -}}
+requests:
+  {{- if $r1 }}
+  cpu: {{ .Values.singleuser.cpu.guarantee }}
+  {{- end }}
+  {{- if $r2 }}
+  memory: {{ .Values.singleuser.memory.guarantee }}
+  {{- end }}
+  {{- if $r3 }}
+  {{- range $key, $value := .Values.singleuser.extraResource.guarantees }}
+  {{ $key | quote }}: {{ $value | quote }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+
+{{- if and $r $l -}}
+{{- println }}
+{{- end }}
+
+{{- if $l -}}
+limits:
+  {{- if $l1 }}
+  cpu: {{ .Values.singleuser.cpu.limit }}
+  {{- end }}
+  {{- if $l2 }}
+  memory: {{ .Values.singleuser.memory.limit }}
+  {{- end }}
+  {{- if $l3 }}
+  {{- range $key, $value := .Values.singleuser.extraResource.limits }}
+  {{ $key | quote }}: {{ $value | quote }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
+{{- /*
+A custom resource request.
+*/}}
+{{- define "jupyterhub.resources" -}}
+{{- if and .Values.scheduling.userPlaceholder.resources (eq .type "user-placeholder") -}}
+{{ .Values.scheduling.userPlaceholder.resources | toYaml | trimSuffix "\n" }}
+{{- else -}}
+{{ include "jupyterhub.default-resources" . }}
+{{- end }}
+{{- end }}

jupyterhub/templates/hub/configmap.yaml

@@ -137,6 +137,7 @@ data:
   singleuser.cloud-metadata: |
     {{- .Values.singleuser.cloudMetadata | toYaml | trimSuffix "\n" | nindent 4 }}
   singleuser.start-timeout: {{ .Values.singleuser.startTimeout | quote }}
+  singleuser.image-spec: {{ .Values.singleuser.image.name }}:{{ .Values.singleuser.image.tag }}
   singleuser.image-pull-policy: {{ .Values.singleuser.image.pullPolicy | quote }}
   {{- if .Values.singleuser.imagePullSecret.enabled }}
   singleuser.image-pull-secret-name: singleuser-image-credentials
@@ -207,16 +208,56 @@ data:
     {{- range $key, $value := .Values.singleuser.extraLabels }}
     {{ $key | quote }}: {{ $value | quote }}
     {{- end }}
+  {{- if .Values.singleuser.storageExtraLabels }}
+  singleuser.storage-extra-labels: |
+    {{- range $key, $value := .Values.singleuser.storageExtraLabels }}
+    {{ $key | quote }}: {{ $value | quote }}
+    {{- end }}
+  {{- end }}
   {{- if .Values.singleuser.extraEnv }}
   singleuser.extra-env: |
     {{- range $key, $value := .Values.singleuser.extraEnv }}
     {{ $key | quote }}: {{ $value | quote }}
     {{- end }}
   {{- end }}
 
+  {{- $_ := merge (dict "podKind" "user") . }}
+  {{- include "jupyterhub.prepareScope" $_ }}
+  {{- if $_.hasTolerations }}
+  singleuser.tolerations: |
+    {{- $_.tolerationsList | nindent 4 }}
+  {{- end }}
+  {{- if $_.nodeAffinityRequired }}
+  singleuser.node-affinity-required: |
+    {{- $_.nodeAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.nodeAffinityPreferred }}
+  singleuser.node-affinity-preferred: |
+    {{- $_.nodeAffinityPreferred | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAffinityRequired }}
+  singleuser.pod-affinity-required: |
+    {{- $_.podAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAffinityPreferred }}
+  singleuser.pod-affinity-preferred: |
+    {{- $_.podAffinityPreferred | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAntiAffinityRequired }}
+  singleuser.pod-anti-affinity-required: |
+    {{- $_.podAntiAffinityRequired | nindent 4 }}
+  {{- end }}
+  {{- if $_.podAntiAffinityPreferred }}
+  singleuser.pod-anti-affinity-preferred: |
+    {{- $_.podAntiAffinityPreferred | nindent 4 }}
+  {{- end }}
+
   {{- if .Values.scheduling.userScheduler.enabled }}
   singleuser.scheduler-name: "{{ .Release.Name }}-user-scheduler"
   {{- end }}
+  {{- if .Values.scheduling.podPriority }}
+  singleuser.priority_class_name: "{{ .Release.Name }}-default-priority"
+  {{- end }}
 
   {{- /* KubeSpawner */}}
   kubespawner.common-labels: |