implement mutation webhooks

- add (optional) mutating wehhook which will add hash on creation of deployments. this will prevent pod restarts when configmaps/secrets already exist
- fix non-404 error handle when loading children
- fix RBACs

charts/wave/templates/clusterrole.yaml

@@ -37,4 +37,10 @@ rules:
       - update
       - patch
       - watch
+  - verbs:
+      - '*'
+    apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
 {{- end }}

charts/wave/templates/deployment.yaml

@@ -32,11 +32,32 @@ spec:
           {{- if .Values.syncPeriod }}
             - --sync-period={{ .Values.syncPeriod }}
           {{- end }}
-          volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 12 }}
+          volumeMounts:
+          {{- if .Values.webhooks.enabled }}
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+          {{- end }} 
+          {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - containerPort: 9443
+              name: webhook-server
+              protocol: TCP
           resources: {{- toYaml .Values.resources | nindent 12 }}
       securityContext: {{ toYaml .Values.securityContext | nindent 8 }}
       serviceAccountName: {{ .Values.serviceAccount.name | default (include "wave-fullname" .) }}
       nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}
       affinity: {{ toYaml .Values.affinity | nindent 8 }}
       tolerations: {{ toYaml .Values.tolerations | nindent 8 }}
-      volumes: {{ toYaml .Values.extraVolumes | nindent 8 }}
+      volumes:
+      {{- if .Values.webhooks.enabled }}
+        - name: cert
+          secret:
+            defaultMode: 420
+            secretName: {{ template "wave-fullname" . }}-webhook-server-cert
+      {{- end }}
+      {{- with .Values.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/wave/templates/webhook.yaml

@@ -0,0 +1,70 @@
+---
+{{- if .Values.webhooks.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration'
+  annotations:
+    cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert'
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-deployment
+    failurePolicy: Ignore
+    name: deployments.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - deployments
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-statefulset
+    failurePolicy: Ignore
+    name: statefulsets.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - statefulsets
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-daemonset
+    failurePolicy: Ignore
+    name: daemonsets.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - daemonsets
+    sideEffects: NoneOnDryRun
+{{- end }}

charts/wave/templates/webhook_certificate.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.webhooks.enabled }}
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "wave-fullname" . }}-serving-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  dnsNames:
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  secretName: {{ template "wave-fullname" . }}-webhook-server-cert
+{{- end }}

charts/wave/templates/webhook_service.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.webhooks.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "wave-fullname" . }}-webhook-service
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{ include "wave-labels.chart" . | nindent 4 }}
+spec:
+  ports:
+    - port: 443
+      targetPort: 9443
+  selector:
+    {{ include "wave-labels.chart" . | nindent 4 }}
+{{- end }}

charts/wave/values.yaml

@@ -43,6 +43,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+webhooks:
+  enabled: false
+
 # Period for reconciliation
 # syncPeriod: 5m
 

cmd/manager/main.go

@@ -27,7 +27,11 @@ import (
 
 	"github.com/wave-k8s/wave/pkg/apis"
 	"github.com/wave-k8s/wave/pkg/controller"
-	"github.com/wave-k8s/wave/pkg/webhook"
+	"github.com/wave-k8s/wave/pkg/controller/daemonset"
+	"github.com/wave-k8s/wave/pkg/controller/deployment"
+	"github.com/wave-k8s/wave/pkg/controller/statefulset"
+	k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
+
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -70,6 +74,9 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	setupLog.Info("setting up manager")
 	mgr, err := manager.New(cfg, manager.Options{
+		WebhookServer: k8swebhook.NewServer(k8swebhook.Options{
+			Port: 9443,
+		}),
 		LeaderElection:          *leaderElection,
 		LeaderElectionID:        *leaderElectionID,
 		LeaderElectionNamespace: *leaderElectionNamespace,
@@ -98,9 +105,18 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("setting up webhooks")
-	if err := webhook.AddToManager(mgr); err != nil {
-		setupLog.Error(err, "unable to register webhooks to the manager")
+	if err := deployment.AddDeploymentWebhook(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "Deployment")
+		os.Exit(1)
+	}
+
+	if err := statefulset.AddStatefulSetWebhook(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet")
+		os.Exit(1)
+	}
+
+	if err := daemonset.AddDaemonSetWebhook(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet")
 		os.Exit(1)
 	}
 

config/webhook/manifests.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-daemonset
+  failurePolicy: Ignore
+  name: daemonsets.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - daemonsets
+  sideEffects: NoneOnDryRun
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-deployment
+  failurePolicy: Ignore
+  name: deployments.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - deployments
+  sideEffects: NoneOnDryRun
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-statefulset
+  failurePolicy: Ignore
+  name: statefulsets.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - statefulsets
+  sideEffects: NoneOnDryRun

pkg/controller/daemonset/daemonset_controller_suite_test.go

@@ -33,6 +33,9 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	admissionv1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var cfg *rest.Config
@@ -47,8 +50,51 @@ var t *envtest.Environment
 var testCtx, testCancel = context.WithCancel(context.Background())
 
 var _ = BeforeSuite(func() {
+	failurePolicy := admissionv1.Ignore
+	sideEffects := admissionv1.SideEffectClassNone
+	webhookPath := "/mutate-apps-v1-daemonset"
+	webhookInstallOptions := envtest.WebhookInstallOptions{
+		MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "daemonset-operator",
+				},
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "MutatingWebhookConfiguration",
+					APIVersion: "admissionregistration.k8s.io/v1",
+				},
+				Webhooks: []admissionv1.MutatingWebhook{
+					{
+						Name:                    "daemonsets.wave.pusher.com",
+						AdmissionReviewVersions: []string{"v1"},
+						FailurePolicy:           &failurePolicy,
+						ClientConfig: admissionv1.WebhookClientConfig{
+							Service: &admissionv1.ServiceReference{
+								Path: &webhookPath,
+							},
+						},
+						Rules: []admissionv1.RuleWithOperations{
+							{
+								Operations: []admissionv1.OperationType{
+									admissionv1.Create,
+									admissionv1.Update,
+								},
+								Rule: admissionv1.Rule{
+									APIGroups:   []string{"apps"},
+									APIVersions: []string{"v1"},
+									Resources:   []string{"daemonsets"},
+								},
+							},
+						},
+						SideEffects: &sideEffects,
+					},
+				},
+			},
+		},
+	}
 	t = &envtest.Environment{
-		CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")},
+		WebhookInstallOptions: webhookInstallOptions,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crds")},
 	}
 
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

pkg/controller/daemonset/daemonset_controller_test.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	webhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
 var _ = Describe("DaemonSet controller Suite", func() {
@@ -94,6 +95,11 @@ var _ = Describe("DaemonSet controller Suite", func() {
 			Metrics: metricsserver.Options{
 				BindAddress: "0",
 			},
+			WebhookServer: webhook.NewServer(webhook.Options{
+				Host:    t.WebhookInstallOptions.LocalServingHost,
+				Port:    t.WebhookInstallOptions.LocalServingPort,
+				CertDir: t.WebhookInstallOptions.LocalServingCertDir,
+			}),
 		})
 		Expect(err).NotTo(HaveOccurred())
 		var cerr error
@@ -106,6 +112,10 @@ var _ = Describe("DaemonSet controller Suite", func() {
 		recFn, requestsStart, requests = SetupTestReconcile(r)
 		Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred())
 
+		// register mutating pod webhook
+		err = AddDaemonSetWebhook(mgr)
+		Expect(err).ToNot(HaveOccurred())
+
 		testCtx, testCancel = context.WithCancel(context.Background())
 		go Run(testCtx, mgr)
 
@@ -205,7 +215,6 @@ var _ = Describe("DaemonSet controller Suite", func() {
 					clearReconciled()
 					m.Update(daemonset, removeContainer2).Should(Succeed())
 					waitForDaemonSetReconciled(daemonset)
-					waitForDaemonSetReconciled(daemonset)
 
 					// Get the updated DaemonSet
 					m.Get(daemonset, timeout).Should(Succeed())