Preset: scheduling.nodeAffinity.matchNodePurpose

jupyterhub · Sep 11, 2018 · 61ea865 · 61ea865
1 parent f554fd9
commit 61ea865
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 46 deletions.
diff --git a/jupyterhub/schema.yaml b/jupyterhub/schema.yaml
@@ -570,3 +570,51 @@ properties:
                 type:
                   - string
                   - "null"
+      corePods:
+        type: object
+        description: |
+          These settings influence the core pods like the hub, proxy and
+          user-scheduler pods.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if core pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=core
+                  ```
+      userPods:
+        type: object
+        description: |
+          These settings influence the user pods like the user-placeholder,
+          user-dummy and actual user pods named like jupyter-someusername.
+        properties:
+          nodeAffinity:
+            type: object
+            description: |
+              Where should pods be scheduled? Perhaps on nodes with a certain
+              label is preferred or even required?
+            properties:
+              matchNodePurpose:
+                type: string
+                enum:
+                  - ignore
+                  - prefer
+                  - require
+                description: |
+                  Decide if user pods *ignore*, *prefer* or *require* to
+                  schedule on nodes with this label:
+                  ```
+                  hub.jupyter.org/node-purpose=user
+                  ```
diff --git a/jupyterhub/templates/hub/deployment.yaml b/jupyterhub/templates/hub/deployment.yaml
@@ -31,26 +31,15 @@ spec:
         {{- .Values.hub.annotations | toYaml | trimSuffix "\n" | nindent 8 }}
         {{- end }}
     spec:
-      nodeSelector: {{ toJson .Values.hub.nodeSelector }}
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['proxy']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
       {{- $_ := merge (dict "podKind" "core") . }}
       {{- $dummy := include "jupyterhub.prepareScope" $_ }}
       {{- if $_.hasTolerations }}
       {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
       {{- end }}
+      nodeSelector: {{ toJson .Values.hub.nodeSelector }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       volumes:
         - name: config
           configMap:

diff --git a/jupyterhub/templates/proxy/autohttps/deployment.yaml b/jupyterhub/templates/proxy/autohttps/deployment.yaml
@@ -36,27 +36,16 @@ spec:
       {{- if .Values.rbac.enabled }}
       serviceAccountName: autohttps
       {{- end }}
-      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
       terminationGracePeriodSeconds: 60
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['hub']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
       {{- $_ := merge (dict "podKind" "core") . }}
       {{- $dummy := include "jupyterhub.prepareScope" $_ }}
       {{- if $_.hasTolerations }}
       {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
       {{- end }}
+      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       containers:
         - name: nginx
           image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}"

diff --git a/jupyterhub/templates/proxy/deployment.yaml b/jupyterhub/templates/proxy/deployment.yaml
@@ -30,27 +30,16 @@ spec:
         {{- .Values.proxy.annotations | toYaml | trimSuffix "\n" | nindent 8 }}
         {{- end }}
     spec:
-      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
       terminationGracePeriodSeconds: 60
-      affinity:
-        podAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchExpressions:
-                    - key: component
-                      operator: In
-                      values: ['hub']
-                    - key: release
-                      operator: In
-                      values: [{{ .Release.Name | quote }}]
       {{- $_ := merge (dict "podKind" "core") . }}
       {{- $dummy := include "jupyterhub.prepareScope" $_ }}
       {{- if $_.hasTolerations }}
       {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
       {{- end }}
+      nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       {{- if $manualHTTPSwithsecret }}
       volumes:
         - name: tls-secret

diff --git a/jupyterhub/templates/scheduling/_scheduling-helpers.tpl b/jupyterhub/templates/scheduling/_scheduling-helpers.tpl
@@ -11,6 +11,14 @@
 {{- $dummy := set . "component" (include "jupyterhub.componentLabel" .) }}
 
 {{- /* Fetch relevant information */}}
+{{- if .podKind }}
+{{- if eq .podKind "core" -}}
+{{- $dummy := set . "matchNodePurpose" .Values.scheduling.corePods.nodeAffinity.matchNodePurpose }}
+{{- else if eq .podKind "user" -}}
+{{- $dummy := set . "matchNodePurpose" .Values.scheduling.userPods.nodeAffinity.matchNodePurpose }}
+{{- end }}
+{{- end }}
+
 {{- $dummy := set . "tolerationsList" (include "jupyterhub.tolerationsList" .) }}
 {{- $dummy := set . "tolerations" (include "jupyterhub.tolerations" .) }}
 {{- $dummy := set . "hasTolerations" .tolerations }}
@@ -25,6 +33,7 @@
 {{- $dummy := set . "hasPodAffinity" (or .podAffinityRequired .podAffinityPreferred) }}
 {{- $dummy := set . "hasPodAntiAffinity" (or .podAntiAffinityRequired .podAntiAffinityPreferred) }}
 {{- $dummy := set . "hasAffinity" (or .hasNodeAffinity .hasPodAffinity .hasPodAntiAffinity) }}
+
 {{- end }}
 
 
@@ -60,12 +69,26 @@ tolerations:
 
 
 {{- define "jupyterhub.nodeAffinityRequired" -}}
+{{- if eq .matchNodePurpose "require" -}}
+- matchExpressions:
+  - key: hub.jupyter.org/node-purpose
+    operator: In
+    values: [{{ .podKind }}]
+{{- end }}
 {{- if .Values.singleuser.extraNodeAffinity.required -}}
 {{- .Values.singleuser.extraNodeAffinity.required | toYaml | trimSuffix "\n" | nindent 0 }}
 {{- end }}
 {{- end }}
 
 {{- define "jupyterhub.nodeAffinityPreferred" -}}
+{{- if eq .matchNodePurpose "prefer" -}}
+- weight: 100
+  preference:
+    matchExpressions:
+      - key: hub.jupyter.org/node-purpose
+        operator: In
+        values: [{{ .podKind }}]
+{{- end }}
 {{- if .Values.singleuser.extraNodeAffinity.preferred -}}
 {{- .Values.singleuser.extraNodeAffinity.preferred | toYaml | trimSuffix "\n" | nindent 0 }}
 {{- end }}

diff --git a/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml b/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml
@@ -29,6 +29,9 @@ spec:
       {{- include "jupyterhub.tolerations" $_ | nindent 6 }}
       {{- end }}
       nodeSelector: {{ toJson .Values.scheduling.userScheduler.nodeSelector }}
+      {{- if $_.hasAffinity }}
+      {{- include "jupyterhub.affinity" $_ | nindent 6 }}
+      {{- end }}
       containers:
         - name: user-scheduler
           image: {{ include "jupyterhub.scheduler.image" . }}

diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml
@@ -230,6 +230,12 @@ scheduling:
     pdb:
       enabled: true
       minAvailable: 1
+  corePods:
+    nodeAffinity:
+      matchNodePurpose: prefer
+  userPods:
+    nodeAffinity:
+      matchNodePurpose: prefer
 
 
 prePuller:

diff --git a/tools/templates/lint-and-validate-values.yaml b/tools/templates/lint-and-validate-values.yaml
@@ -105,7 +105,7 @@ proxy:
     enabled: true
   https:
     enabled: true
-    type: manual
+    type: letsencrypt
     #type: letsencrypt, manual, secret
     letsencrypt:
       contactEmail: 'e@domain.com'
@@ -229,6 +229,12 @@ scheduling:
     image:
       name: gcr.io/google_containers/kube-scheduler-amd64
       tag: v1.11.2
+  corePods:
+    nodeAffinity:
+      matchNodePurpose: require
+  userPods:
+    nodeAffinity:
+      matchNodePurpose: require
 
 
 prePuller: