diff --git a/charts/wave/templates/clusterrole.yaml b/charts/wave/templates/clusterrole.yaml index 2adcb98a..4858de6e 100644 --- a/charts/wave/templates/clusterrole.yaml +++ b/charts/wave/templates/clusterrole.yaml @@ -37,4 +37,10 @@ rules: - update - patch - watch + - verbs: + - '*' + apiGroups: + - coordination.k8s.io + resources: + - leases {{- end }} diff --git a/charts/wave/templates/deployment.yaml b/charts/wave/templates/deployment.yaml index 7a1f3b1c..3e181c95 100644 --- a/charts/wave/templates/deployment.yaml +++ b/charts/wave/templates/deployment.yaml @@ -32,11 +32,32 @@ spec: {{- if .Values.syncPeriod }} - --sync-period={{ .Values.syncPeriod }} {{- end }} - volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 12 }} + volumeMounts: + {{- if .Values.webhooks.enabled }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP resources: {{- toYaml .Values.resources | nindent 12 }} securityContext: {{ toYaml .Values.securityContext | nindent 8 }} serviceAccountName: {{ .Values.serviceAccount.name | default (include "wave-fullname" .) }} nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }} affinity: {{ toYaml .Values.affinity | nindent 8 }} tolerations: {{ toYaml .Values.tolerations | nindent 8 }} - volumes: {{ toYaml .Values.extraVolumes | nindent 8 }} + volumes: + {{- if .Values.webhooks.enabled }} + - name: cert + secret: + defaultMode: 420 + secretName: {{ template "wave-fullname" . }}-webhook-server-cert + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook.yaml b/charts/wave/templates/webhook.yaml new file mode 100644 index 00000000..e81290ea --- /dev/null +++ b/charts/wave/templates/webhook.yaml @@ -0,0 +1,70 @@ +--- +{{- if .Values.webhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration' + annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert' +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: deployments.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-statefulset + failurePolicy: Ignore + name: statefulsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-daemonset + failurePolicy: Ignore + name: daemonsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + sideEffects: NoneOnDryRun +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_certificate.yaml b/charts/wave/templates/webhook_certificate.yaml new file mode 100644 index 00000000..5d63f8ba --- /dev/null +++ b/charts/wave/templates/webhook_certificate.yaml @@ -0,0 +1,25 @@ +{{- if .Values.webhooks.enabled }} +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "wave-fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "wave-fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ template "wave-fullname" . }}-selfsigned-issuer + secretName: {{ template "wave-fullname" . }}-webhook-server-cert +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_service.yaml b/charts/wave/templates/webhook_service.yaml new file mode 100644 index 00000000..a3b2532a --- /dev/null +++ b/charts/wave/templates/webhook_service.yaml @@ -0,0 +1,15 @@ +{{- if .Values.webhooks.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "wave-fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + labels: + {{ include "wave-labels.chart" . | nindent 4 }} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + {{ include "wave-labels.chart" . | nindent 4 }} +{{- end }} diff --git a/charts/wave/values.yaml b/charts/wave/values.yaml index e9ee1951..d1b68088 100644 --- a/charts/wave/values.yaml +++ b/charts/wave/values.yaml @@ -43,6 +43,9 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +webhooks: + enabled: false + # Period for reconciliation # syncPeriod: 5m diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 3065b755..177dc62e 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -27,7 +27,11 @@ import ( "github.com/wave-k8s/wave/pkg/apis" "github.com/wave-k8s/wave/pkg/controller" - "github.com/wave-k8s/wave/pkg/webhook" + "github.com/wave-k8s/wave/pkg/controller/daemonset" + "github.com/wave-k8s/wave/pkg/controller/deployment" + "github.com/wave-k8s/wave/pkg/controller/statefulset" + k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -70,6 +74,9 @@ func main() { // Create a new Cmd to provide shared dependencies and start components setupLog.Info("setting up manager") mgr, err := manager.New(cfg, manager.Options{ + WebhookServer: k8swebhook.NewServer(k8swebhook.Options{ + Port: 9443, + }), LeaderElection: *leaderElection, LeaderElectionID: *leaderElectionID, LeaderElectionNamespace: *leaderElectionNamespace, @@ -98,9 +105,18 @@ func main() { os.Exit(1) } - setupLog.Info("setting up webhooks") - if err := webhook.AddToManager(mgr); err != nil { - setupLog.Error(err, "unable to register webhooks to the manager") + if err := deployment.AddDeploymentWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Deployment") + os.Exit(1) + } + + if err := statefulset.AddStatefulSetWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet") + os.Exit(1) + } + + if err := daemonset.AddDaemonSetWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet") os.Exit(1) } diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index e69de29b..10be8448 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-daemonset + failurePolicy: Ignore + name: daemonsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + sideEffects: NoneOnDryRun +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: deployments.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: NoneOnDryRun +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-statefulset + failurePolicy: Ignore + name: statefulsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: NoneOnDryRun diff --git a/pkg/controller/daemonset/daemonset_controller_suite_test.go b/pkg/controller/daemonset/daemonset_controller_suite_test.go index 54ad0c49..ba864bcd 100644 --- a/pkg/controller/daemonset/daemonset_controller_suite_test.go +++ b/pkg/controller/daemonset/daemonset_controller_suite_test.go @@ -33,6 +33,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -47,8 +50,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-daemonset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daemonset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "daemonsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"daemonsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go index 11fb86f4..8e2c9da2 100644 --- a/pkg/controller/daemonset/daemonset_controller_test.go +++ b/pkg/controller/daemonset/daemonset_controller_test.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("DaemonSet controller Suite", func() { @@ -94,6 +95,11 @@ var _ = Describe("DaemonSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -106,6 +112,10 @@ var _ = Describe("DaemonSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddDaemonSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -205,7 +215,6 @@ var _ = Describe("DaemonSet controller Suite", func() { clearReconciled() m.Update(daemonset, removeContainer2).Should(Succeed()) waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) // Get the updated DaemonSet m.Get(daemonset, timeout).Should(Succeed()) diff --git a/pkg/controller/daemonset/daemonset_webhook.go b/pkg/controller/daemonset/daemonset_webhook.go new file mode 100644 index 00000000..22605121 --- /dev/null +++ b/pkg/controller/daemonset/daemonset_webhook.go @@ -0,0 +1,39 @@ +package daemonset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-daemonset,mutating=true,failurePolicy=ignore,groups=apps,resources=daemonsets,verbs=create;update,versions=v1,name=daemonsets.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type DaemonSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDaemonSetWebhook(obj.(*appsv1.DaemonSet), request.DryRun) + return err +} + +func AddDaemonSetWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter( + &DaemonSetWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/controller/deployment/deployment_controller_suite_test.go b/pkg/controller/deployment/deployment_controller_suite_test.go index 2e36804b..19f34726 100644 --- a/pkg/controller/deployment/deployment_controller_suite_test.go +++ b/pkg/controller/deployment/deployment_controller_suite_test.go @@ -34,6 +34,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -48,8 +51,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-deployment" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "deployment-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "deployments.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"deployments"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/deployment/deployment_controller_test.go b/pkg/controller/deployment/deployment_controller_test.go index 39c43ec2..2ad6b645 100644 --- a/pkg/controller/deployment/deployment_controller_test.go +++ b/pkg/controller/deployment/deployment_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("Deployment controller Suite", func() { @@ -101,6 +102,11 @@ var _ = Describe("Deployment controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -113,6 +119,10 @@ var _ = Describe("Deployment controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddDeploymentWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -232,7 +242,6 @@ var _ = Describe("Deployment controller Suite", func() { clearReconciled() m.Update(deployment, removeContainer2).Should(Succeed()) waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) // Get the updated Deployment m.Get(deployment, timeout).Should(Succeed()) diff --git a/pkg/controller/deployment/deployment_webhook.go b/pkg/controller/deployment/deployment_webhook.go new file mode 100644 index 00000000..f75a126a --- /dev/null +++ b/pkg/controller/deployment/deployment_webhook.go @@ -0,0 +1,39 @@ +package deployment + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=ignore,groups=apps,resources=deployments,verbs=create;update,versions=v1,name=deployments.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type DeploymentWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DeploymentWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDeploymentWebhook(obj.(*appsv1.Deployment), request.DryRun) + return err +} + +func AddDeploymentWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter( + &DeploymentWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/controller/statefulset/statefulset_controller_suite_test.go b/pkg/controller/statefulset/statefulset_controller_suite_test.go index 689245de..6c5ba172 100644 --- a/pkg/controller/statefulset/statefulset_controller_suite_test.go +++ b/pkg/controller/statefulset/statefulset_controller_suite_test.go @@ -35,6 +35,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -49,8 +52,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-statefulset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "statefulset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "statefulsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"statefulsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index 6e8a2221..d62b63bc 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("StatefulSet controller Suite", func() { @@ -95,6 +96,11 @@ var _ = Describe("StatefulSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) @@ -109,6 +115,10 @@ var _ = Describe("StatefulSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddStatefulSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -210,7 +220,6 @@ var _ = Describe("StatefulSet controller Suite", func() { clearReconciled() m.Update(statefulset, removeContainer2).Should(Succeed()) waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) // Get the updated StatefulSet m.Get(statefulset, timeout).Should(Succeed()) diff --git a/pkg/controller/statefulset/statefulset_webhook.go b/pkg/controller/statefulset/statefulset_webhook.go new file mode 100644 index 00000000..40a41a9e --- /dev/null +++ b/pkg/controller/statefulset/statefulset_webhook.go @@ -0,0 +1,39 @@ +package statefulset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-statefulset,mutating=true,failurePolicy=ignore,groups=apps,resources=statefulsets,verbs=create;update,versions=v1,name=statefulsets.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type StatefulSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *StatefulSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleStatefulSetWebhook(obj.(*appsv1.StatefulSet), request.DryRun) + return err +} + +func AddStatefulSetWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter( + &StatefulSetWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/core/children.go b/pkg/core/children.go index 9b109979..c4f957ef 100644 --- a/pkg/core/children.go +++ b/pkg/core/children.go @@ -22,6 +22,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,10 +39,19 @@ type configMetadata struct { keys map[string]struct{} } +type NotFoundError struct { + string +} + +func (e *NotFoundError) Error() string { + return e.string +} + // getResult is returned from the getObject method as a helper struct to be // passed into a channel type getResult struct { err error + notFound bool obj Object metadata configMetadata } @@ -68,11 +78,16 @@ func (h *Handler) getCurrentChildren(obj podController) ([]configObject, error) // Range over and collect results from the gets var errs []string + var notFoundErrs []string var children []configObject for i := 0; i < len(configMaps)+len(secrets); i++ { result := <-resultsChan if result.err != nil { - errs = append(errs, result.err.Error()) + if result.notFound { + notFoundErrs = append(notFoundErrs, result.err.Error()) + } else { + errs = append(errs, result.err.Error()) + } } if result.obj != nil { children = append(children, configObject{ @@ -89,6 +104,11 @@ func (h *Handler) getCurrentChildren(obj podController) ([]configObject, error) return []configObject{}, fmt.Errorf("error(s) encountered when geting children: %s", strings.Join(errs, ", ")) } + // If there we did not find required elements + if len(notFoundErrs) > 0 { + return []configObject{}, &NotFoundError{fmt.Sprintf("required children not found: %s", strings.Join(notFoundErrs, ", "))} + } + // No errors, return the list of children return children, nil } @@ -219,12 +239,16 @@ func (h *Handler) getObject(namespace, name string, metadata configMetadata, obj objectName := types.NamespacedName{Namespace: namespace, Name: name} err := h.Get(context.TODO(), objectName, obj) if err != nil { - if metadata.required { - return getResult{err: err} + if errors.IsNotFound(err) { + if metadata.required { + return getResult{err: err, notFound: true} + } + return getResult{metadata: metadata, notFound: true} + } else { + return getResult{err: err, notFound: false} } - return getResult{metadata: metadata} } - return getResult{obj: obj, metadata: metadata} + return getResult{obj: obj, metadata: metadata, notFound: false} } // getExistingChildren returns a list of all Secrets and ConfigMaps that are diff --git a/pkg/core/handler.go b/pkg/core/handler.go index 588eddf6..06940de9 100644 --- a/pkg/core/handler.go +++ b/pkg/core/handler.go @@ -48,6 +48,21 @@ func (h *Handler) HandleDeployment(instance *appsv1.Deployment) (reconcile.Resul return h.handlePodController(&deployment{Deployment: instance}) } +// HandleDeploymentWebhook is called by the deployment webhook +func (h *Handler) HandleDeploymentWebhook(instance *appsv1.Deployment, dryRun *bool) error { + return h.updatePodController(&deployment{Deployment: instance}, (dryRun != nil && *dryRun)) +} + +// HandleStatefulSetWebhook is called by the statefulset webhook +func (h *Handler) HandleStatefulSetWebhook(instance *appsv1.StatefulSet, dryRun *bool) error { + return h.updatePodController(&statefulset{StatefulSet: instance}, (dryRun != nil && *dryRun)) +} + +// HandleDaemonSetWebhook is called by the daemonset webhook +func (h *Handler) HandleDaemonSetWebhook(instance *appsv1.DaemonSet, dryRun *bool) error { + return h.updatePodController(&daemonset{DaemonSet: instance}, (dryRun != nil && *dryRun)) +} + // HandleStatefulSet is called by the StatefulSet controller to reconcile StatefulSets func (h *Handler) HandleStatefulSet(instance *appsv1.StatefulSet) (reconcile.Result, error) { return h.handlePodController(&statefulset{StatefulSet: instance}) @@ -74,6 +89,8 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, return reconcile.Result{}, nil } + log.V(5).Info("Reconciling") + // Get all children that the instance currently references current, err := h.getCurrentChildren(instance) if err != nil { @@ -104,3 +121,41 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, } return reconcile.Result{}, nil } + +// handlePodController will only update the hash. Everything else is left to the reconciler. +func (h *Handler) updatePodController(instance podController, dryRun bool) error { + log := logf.Log.WithName("wave").WithValues("namespace", instance.GetNamespace(), "name", instance.GetName(), "dryRun", dryRun) + log.V(5).Info("Running webhook") + + // If the required annotation isn't present, ignore the instance + if !hasRequiredAnnotation(instance) { + return nil + } + + // Get all children that the instance currently references + current, err := h.getCurrentChildren(instance) + if err != nil { + if _, ok := err.(*NotFoundError); ok { + log.V(0).Info("Not all required children found yet. Skipping mutation!", "err", err) + return nil + } else { + return fmt.Errorf("error fetching current children: %v", err) + } + } + + hash, err := calculateConfigHash(current) + if err != nil { + return fmt.Errorf("error calculating configuration hash: %v", err) + } + + // Update the desired state of the Deployment + oldHash := getConfigHash(instance) + setConfigHash(instance, hash) + + if !dryRun && oldHash != hash { + log.V(0).Info("Updating instance hash", "hash", oldHash) + h.recorder.Eventf(instance.GetApiObject(), corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) + } + + return nil +} diff --git a/pkg/core/hash.go b/pkg/core/hash.go index 85befda8..c179f73b 100644 --- a/pkg/core/hash.go +++ b/pkg/core/hash.go @@ -105,7 +105,7 @@ func getSecretData(child configObject) map[string][]byte { return keyData } -// setConfigHash upates the configuration hash of the given Deployment to the +// setConfigHash updates the configuration hash of the given Deployment to the // given string func setConfigHash(obj podController, hash string) { // Get the existing annotations @@ -120,3 +120,9 @@ func setConfigHash(obj podController, hash string) { podTemplate.SetAnnotations(annotations) obj.SetPodTemplate(podTemplate) } + +// getConfigHash return the config hash string +func getConfigHash(obj podController) string { + podTemplate := obj.GetPodTemplate() + return podTemplate.GetAnnotations()[ConfigHashAnnotation] +}