Merge pull request #793 from jvanz/tolerations-issue752

feat: policy server deployment tolerations.

api/policies/v1/policyserver_types.go

@@ -113,6 +113,11 @@ type PolicyServerSpec struct {
  // otherwise to an implementation-defined value
  // +optional
  Requests corev1.ResourceList `json:"requests,omitempty"`
+
+ // Tolerations describes the policy server pod's tolerations. It can be
+ // user to ensure that the policy server pod is not scheduled onto a
+ // node with a taint.
+ Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 }
 
 type ReconciliationTransitionReason string

config/crd/bases/policies.kubewarden.io_policyservers.yaml

@@ -1580,6 +1580,48 @@ spec:
  `sources.yaml`. Reference for `sources.yaml` is found in the Kubewarden
  documentation in the reference section.
  type: object
+ tolerations:
+ description: |-
+ Tolerations describes the policy server pod's tolerations. It can be
+ user to ensure that the policy server pod is not scheduled onto a
+ node with a taint.
+ items:
+ description: |-
+ The pod this Toleration is attached to tolerates any taint that matches
+ the triple <key,value,effect> using the matching operator <operator>.
+ properties:
+ effect:
+ description: |-
+ Effect indicates the taint effect to match. Empty means match all taint effects.
+ When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: |-
+ Key is the taint key that the toleration applies to. Empty means match all taint keys.
+ If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: |-
+ Operator represents a key's relationship to the value.
+ Valid operators are Exists and Equal. Defaults to Equal.
+ Exists is equivalent to wildcard for value, so that a pod can
+ tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: |-
+ TolerationSeconds represents the period of time the toleration (which must be
+ of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+ it is not set, which means tolerate the taint forever (do not evict). Zero and
+ negative values will be treated as 0 (evict immediately) by the system.
+ format: int64
+ type: integer
+ value:
+ description: |-
+ Value is the taint value the toleration matches to.
+ If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ type: object
+ type: array
  verificationConfig:
  description: |-
  Name of VerificationConfig configmap in the same namespace, containing

internal/controller/policyserver_controller_deployment.go

@@ -168,6 +168,8 @@ func (r *PolicyServerReconciler) updatePolicyServerDeployment(policyServer *poli
  SecurityContext: podSecurityContext,
  Containers: []corev1.Container{admissionContainer},
  ServiceAccountName: policyServer.Spec.ServiceAccountName,
+ Tolerations: policyServer.Spec.Tolerations,
+ Affinity: &policyServer.Spec.Affinity,
  Volumes: []corev1.Volume{
  {
  Name: policyStoreVolume,
@@ -295,10 +297,6 @@ func (r *PolicyServerReconciler) adaptDeploymentSettingsForPolicyServer(policySe
  },
  )
  }
-
- if emptyAffinity := (corev1.Affinity{}); policyServer.Spec.Affinity != emptyAffinity {
- policyServerDeployment.Spec.Template.Spec.Affinity = &policyServer.Spec.Affinity
- }
 }
 
 func envVarsContainVariable(envVars []corev1.EnvVar, envVarName string) int {

internal/controller/policyserver_controller_test.go

@@ -250,6 +250,46 @@ var _ = Describe("PolicyServer controller", func() {
  })
 
  When("creating a PolicyServer", func() {
+ It("should use the policy server tolerations configuration in the policy server deployment", func() {
+ tolerationSeconds := int64(10)
+ policyServer := policyServerFactory(policyServerName)
+ policyServer.Spec.Tolerations = []corev1.Toleration{{
+ Key: "key1",
+ Operator: corev1.TolerationOpEqual,
+ Value: "value1",
+ Effect: corev1.TaintEffectNoSchedule,
+ TolerationSeconds: nil,
+ }, {
+ Key: "key2",
+ Operator: corev1.TolerationOpEqual,
+ Value: "value2",
+ Effect: corev1.TaintEffectNoExecute,
+ TolerationSeconds: &tolerationSeconds,
+ }}
+ createPolicyServerAndWaitForItsService(policyServer)
+ deployment, err := getTestPolicyServerDeployment(policyServerName)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(deployment.Spec.Template.Spec.Tolerations).To(MatchAllElements(func(element interface{}) string {
+ toleration, _ := element.(corev1.Toleration)
+ return toleration.Key
+ }, Elements{
+ "key1": MatchAllFields(Fields{
+ "Key": Equal("key1"),
+ "Operator": Equal(corev1.TolerationOpEqual),
+ "Value": Equal("value1"),
+ "Effect": Equal(corev1.TaintEffectNoSchedule),
+ "TolerationSeconds": BeNil(),
+ }),
+ "key2": MatchAllFields(Fields{
+ "Key": Equal("key2"),
+ "Operator": Equal(corev1.TolerationOpEqual),
+ "Value": Equal("value2"),
+ "Effect": Equal(corev1.TaintEffectNoExecute),
+ "TolerationSeconds": PointTo(Equal(tolerationSeconds)),
+ }),
+ }))
+ })
+
  It("should use the policy server affinity configuration in the policy server deployment", func() {
  policyServer := policyServerFactory(policyServerName)
  policyServer.Spec.Affinity = corev1.Affinity{
@@ -311,22 +351,27 @@ var _ = Describe("PolicyServer controller", func() {
  "SeccompProfile": BeNil(),
  })),
  })))
- By("checking the deployment pod security context")
- Expect(deployment.Spec.Template.Spec.SecurityContext).To(PointTo(MatchFields(IgnoreExtras, Fields{
- "SELinuxOptions": BeNil(),
- "WindowsOptions": BeNil(),
- "RunAsUser": BeNil(),
- "RunAsGroup": BeNil(),
- "RunAsNonRoot": BeNil(),
- "SupplementalGroups": BeNil(),
- "FSGroup": BeNil(),
- "Sysctls": BeNil(),
- "FSGroupChangePolicy": BeNil(),
- "SeccompProfile": BeNil(),
- })))
-
- By("checking the deployment affinity")
- Expect(deployment.Spec.Template.Spec.Affinity).To(BeNil())
+ By("checking the deployment spec")
+ Expect(deployment.Spec.Template.Spec).To(MatchFields(IgnoreExtras, Fields{
+ "Tolerations": BeEmpty(),
+ "SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{
+ "SELinuxOptions": BeNil(),
+ "WindowsOptions": BeNil(),
+ "RunAsUser": BeNil(),
+ "RunAsGroup": BeNil(),
+ "RunAsNonRoot": BeNil(),
+ "SupplementalGroups": BeNil(),
+ "FSGroup": BeNil(),
+ "Sysctls": BeNil(),
+ "FSGroupChangePolicy": BeNil(),
+ "SeccompProfile": BeNil(),
+ })),
+ "Affinity": PointTo(MatchAllFields(Fields{
+ "NodeAffinity": BeNil(),
+ "PodAffinity": BeNil(),
+ "PodAntiAffinity": BeNil(),
+ })),
+ }))
  })
 
  It("should create the policy server deployment and use the user defined security contexts", func() {