Injector scheduler options (hashicorp#234)

Adds affinity, tolerations, and nodeSelector options for the
injector deployment that are separate from those options on the vault
server statefulset.

Co-authored-by: Sergei Shishov <sergei.shishov@dubizzle.com>

templates/_helpers.tpl

@@ -212,6 +212,16 @@ Set's the affinity for pod placement when running in standalone and HA modes.
   {{ end }}
 {{- end -}}
 
+{{/*
+Sets the injector affinity for pod placement
+*/}}
+{{- define "injector.affinity" -}}
+  {{- if .Values.injector.affinity }}
+      affinity:
+        {{ tpl .Values.injector.affinity . | nindent 8 | trim }}
+  {{ end }}
+{{- end -}}
+
 {{/*
 Set's the toleration for pod placement when running in standalone and HA modes.
 */}}
@@ -222,6 +232,16 @@ Set's the toleration for pod placement when running in standalone and HA modes.
   {{- end }}
 {{- end -}}
 
+{{/*
+Sets the injector toleration for pod placement
+*/}}
+{{- define "injector.tolerations" -}}
+  {{- if .Values.injector.tolerations }}
+      tolerations:
+        {{ tpl .Values.injector.tolerations . | nindent 8 | trim }}
+  {{- end }}
+{{- end -}}
+
 {{/*
 Set's the node selector for pod placement when running in standalone and HA modes.
 */}}
@@ -232,6 +252,16 @@ Set's the node selector for pod placement when running in standalone and HA mode
   {{- end }}
 {{- end -}}
 
+{{/*
+Sets the injector node selector for pod placement
+*/}}
+{{- define "injector.nodeselector" -}}
+  {{- if .Values.injector.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.injector.nodeSelector . | indent 8 | trim }}
+  {{- end }}
+{{- end -}}
+
 {{/*
 Sets extra pod annotations
 */}}

templates/injector-deployment.yaml

@@ -24,6 +24,9 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
         component: webhook
     spec:
+      {{ template "injector.affinity" . }}
+      {{ template "injector.tolerations" . }}
+      {{ template "injector.nodeselector" . }}
       serviceAccountName: "{{ template "vault.fullname" . }}-agent-injector"
       securityContext:
         runAsNonRoot: true

test/unit/injector-deployment.bats

@@ -359,3 +359,69 @@ load _helpers
       yq -r '.[11].value' | tee /dev/stderr)
   [ "${actual}" = "sanitized" ]
 }
+
+#--------------------------------------------------------------------
+# affinity
+
+@test "injector/deployment: affinity not set by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec | .affinity? == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "injector/deployment: affinity can be set" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml  \
+      --set 'injector.affinity=foobar' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.affinity == "foobar"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+#--------------------------------------------------------------------
+# tolerations
+
+@test "injector/deployment: tolerations not set by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec | .tolerations? == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "injector/deployment: tolerations can be set" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml  \
+      --set 'injector.tolerations=foobar' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.tolerations == "foobar"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+#--------------------------------------------------------------------
+# nodeSelector
+
+@test "injector/deployment: nodeSelector is not set by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.nodeSelector' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+}
+
+@test "injector/deployment: nodeSelector can be set" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set 'injector.nodeSelector=testing' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.nodeSelector' | tee /dev/stderr)
+  [ "${actual}" = "testing" ]
+}

test/unit/server-statefulset.bats

@@ -561,6 +561,25 @@ load _helpers
   [ "${actual}" = "0" ]
 }
 
+@test "server/standalone-StatefulSet: affinity is set by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml  \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.affinity["podAntiAffinity"]? != null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/standalone-StatefulSet: affinity can be set" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml  \
+      --set 'server.affinity=foobar' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.affinity == "foobar"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 @test "server/standalone-StatefulSet: tolerations not set by default" {
   cd `chart_dir`
   local actual=$(helm template \

values.yaml

@@ -86,6 +86,23 @@ injector:
   extraEnvironmentVars: {}
     # KUBERNETES_SERVICE_HOST: kubernetes.default.svc
 
+  # Affinity Settings for injector pods
+  # This should be a multi-line string matching the affinity section of a
+  # PodSpec.
+  affinity: null
+
+  # Toleration Settings for injector pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: null
+
+  # nodeSelector labels for injector pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  # Example:
+  # nodeSelector: |
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: null
+
 server:
   # Resource requests, limits, etc. for the server cluster placement. This
   # should map directly to the value of the resources field for a PodSpec.