diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 68bed90f..2815f5b6 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -10,6 +10,9 @@ jobs: version: - v1.21 - v1.29 + setup: + - minimal + - production runs-on: ubuntu-latest name: test on minikube steps: @@ -19,4 +22,4 @@ jobs: with: kubernetes-version: ${{ matrix.version }} - name: Build and run wave - run: hack/run-test-in-minikube.sh + run: hack/run-test-in-minikube.sh ${{ matrix.setup }} diff --git a/charts/wave/templates/clusterrole.yaml b/charts/wave/templates/clusterrole.yaml index 2adcb98a..4858de6e 100644 --- a/charts/wave/templates/clusterrole.yaml +++ b/charts/wave/templates/clusterrole.yaml @@ -37,4 +37,10 @@ rules: - update - patch - watch + - verbs: + - '*' + apiGroups: + - coordination.k8s.io + resources: + - leases {{- end }} diff --git a/charts/wave/templates/deployment.yaml b/charts/wave/templates/deployment.yaml index b4296ca7..ab26187c 100644 --- a/charts/wave/templates/deployment.yaml +++ b/charts/wave/templates/deployment.yaml @@ -32,12 +32,36 @@ spec: {{- if .Values.syncPeriod }} - --sync-period={{ .Values.syncPeriod }} {{- end }} - volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- if .Values.webhooks.enabled }} + - --enable-webhooks=true + {{- end }} + volumeMounts: + {{- if .Values.webhooks.enabled }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP resources: {{- toYaml .Values.resources | nindent 12 }} securityContext: {{ toYaml .Values.securityContext | nindent 8 }} serviceAccountName: {{ .Values.serviceAccount.name | default (include "wave-fullname" .) }} nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }} affinity: {{ toYaml .Values.affinity | nindent 8 }} tolerations: {{ toYaml .Values.tolerations | nindent 8 }} + volumes: + {{- if .Values.webhooks.enabled }} + - name: cert + secret: + defaultMode: 420 + secretName: {{ template "wave-fullname" . }}-webhook-server-cert + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} topologySpreadConstraints: {{ toYaml .Values.topologySpreadConstraints | nindent 8 }} - volumes: {{ toYaml .Values.extraVolumes | nindent 8 }} diff --git a/charts/wave/templates/poddisruptionbudget.yaml b/charts/wave/templates/poddisruptionbudget.yaml index 204bcc10..d11588a8 100644 --- a/charts/wave/templates/poddisruptionbudget.yaml +++ b/charts/wave/templates/poddisruptionbudget.yaml @@ -7,6 +7,6 @@ metadata: spec: selector: matchLabels: - {{ include "wave.selectorLabels" . | nindent 6}} + {{ include "wave-labels.chart" . | nindent 6 }} maxUnavailable: 1 {{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook.yaml b/charts/wave/templates/webhook.yaml new file mode 100644 index 00000000..e81290ea --- /dev/null +++ b/charts/wave/templates/webhook.yaml @@ -0,0 +1,70 @@ +--- +{{- if .Values.webhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration' + annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert' +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: deployments.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-statefulset + failurePolicy: Ignore + name: statefulsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-daemonset + failurePolicy: Ignore + name: daemonsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + sideEffects: NoneOnDryRun +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_certificate.yaml b/charts/wave/templates/webhook_certificate.yaml new file mode 100644 index 00000000..5d63f8ba --- /dev/null +++ b/charts/wave/templates/webhook_certificate.yaml @@ -0,0 +1,25 @@ +{{- if .Values.webhooks.enabled }} +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "wave-fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "wave-fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ template "wave-fullname" . }}-selfsigned-issuer + secretName: {{ template "wave-fullname" . }}-webhook-server-cert +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_service.yaml b/charts/wave/templates/webhook_service.yaml new file mode 100644 index 00000000..a3b2532a --- /dev/null +++ b/charts/wave/templates/webhook_service.yaml @@ -0,0 +1,15 @@ +{{- if .Values.webhooks.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "wave-fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + labels: + {{ include "wave-labels.chart" . | nindent 4 }} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + {{ include "wave-labels.chart" . | nindent 4 }} +{{- end }} diff --git a/charts/wave/values.yaml b/charts/wave/values.yaml index add081da..6ec2e4f5 100644 --- a/charts/wave/values.yaml +++ b/charts/wave/values.yaml @@ -47,6 +47,9 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +webhooks: + enabled: false + # Period for reconciliation # syncPeriod: 5m diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 3065b755..8667f369 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -27,7 +27,11 @@ import ( "github.com/wave-k8s/wave/pkg/apis" "github.com/wave-k8s/wave/pkg/controller" - "github.com/wave-k8s/wave/pkg/webhook" + "github.com/wave-k8s/wave/pkg/controller/daemonset" + "github.com/wave-k8s/wave/pkg/controller/deployment" + "github.com/wave-k8s/wave/pkg/controller/statefulset" + k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -42,6 +46,7 @@ var ( leaderElectionNamespace = flag.String("leader-election-namespace", "", "Namespace for the configmap used by the leader election system") syncPeriod = flag.Duration("sync-period", 5*time.Minute, "Reconcile sync period") showVersion = flag.Bool("version", false, "Show version and exit") + enableWebhooks = flag.Bool("enable-webhooks", false, "Enable webhooks") setupLog = ctrl.Log.WithName("setup") ) @@ -69,7 +74,14 @@ func main() { // Create a new Cmd to provide shared dependencies and start components setupLog.Info("setting up manager") + var webhookServer k8swebhook.Server + if *enableWebhooks { + webhookServer = k8swebhook.NewServer(k8swebhook.Options{ + Port: 9443, + }) + } mgr, err := manager.New(cfg, manager.Options{ + WebhookServer: webhookServer, LeaderElection: *leaderElection, LeaderElectionID: *leaderElectionID, LeaderElectionNamespace: *leaderElectionNamespace, @@ -97,11 +109,21 @@ func main() { setupLog.Error(err, "unable to register controllers to the manager") os.Exit(1) } - - setupLog.Info("setting up webhooks") - if err := webhook.AddToManager(mgr); err != nil { - setupLog.Error(err, "unable to register webhooks to the manager") - os.Exit(1) + if *enableWebhooks { + if err := deployment.AddDeploymentWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Deployment") + os.Exit(1) + } + + if err := statefulset.AddStatefulSetWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet") + os.Exit(1) + } + + if err := daemonset.AddDaemonSetWebhook(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet") + os.Exit(1) + } } // Start the Cmd diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index e69de29b..10be8448 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-daemonset + failurePolicy: Ignore + name: daemonsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + sideEffects: NoneOnDryRun +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: deployments.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: NoneOnDryRun +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-v1-statefulset + failurePolicy: Ignore + name: statefulsets.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: NoneOnDryRun diff --git a/hack/production.yaml b/hack/production.yaml new file mode 100644 index 00000000..fe40d0a6 --- /dev/null +++ b/hack/production.yaml @@ -0,0 +1,16 @@ +# production setup used in tests +replicas: 2 + +webhooks: + enabled: true + +pdb: + enabled: true + +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: wave diff --git a/hack/run-test-in-minikube.sh b/hack/run-test-in-minikube.sh index 940ef97f..e10429bb 100755 --- a/hack/run-test-in-minikube.sh +++ b/hack/run-test-in-minikube.sh @@ -33,7 +33,17 @@ eval $(minikube -p minikube docker-env) docker build -f ./Dockerfile -t wave-local:local . echo Installing wave... -helm install wave charts/wave --set image.name=wave-local --set image.tag=local +if [ "$1" = "production" ]; then + # Install cert-manager + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml + while [ "$(kubectl get pods -A | grep -c 'cert-manager')" -le 0 ]; do echo Waiting for \"cert-manager\" to exist; sleep 10; done + while [ "$(kubectl get pods -A | grep 'cert-manager' | grep -cEv 'Running|Completed')" -gt 0 ]; do echo Waiting for \"cert-manager\" to start; sleep 10; done + # Production setup + helm install wave charts/wave --set image.name=wave-local --set image.tag=local -f hack/production.yaml +else + # Default install without values + helm install wave charts/wave --set image.name=wave-local --set image.tag=local +fi while [ "$(kubectl get pods -A | grep -cEv 'Running|Completed')" -gt 1 ]; do echo Waiting for \"cluster\" to start; sleep 10; done diff --git a/pkg/controller/daemonset/daemonset_controller_suite_test.go b/pkg/controller/daemonset/daemonset_controller_suite_test.go index 4ad3d13e..a05cca5e 100644 --- a/pkg/controller/daemonset/daemonset_controller_suite_test.go +++ b/pkg/controller/daemonset/daemonset_controller_suite_test.go @@ -31,6 +31,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -45,8 +48,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-daemonset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daemonset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "daemonsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"daemonsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go index efb9d3a8..a7609769 100644 --- a/pkg/controller/daemonset/daemonset_controller_test.go +++ b/pkg/controller/daemonset/daemonset_controller_test.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("DaemonSet controller Suite", func() { @@ -94,6 +95,11 @@ var _ = Describe("DaemonSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -106,6 +112,10 @@ var _ = Describe("DaemonSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddDaemonSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -163,8 +173,6 @@ var _ = Describe("DaemonSet controller Suite", func() { } clearReconciled() m.Update(daemonset, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForDaemonSetReconciled(daemonset) waitForDaemonSetReconciled(daemonset) // Get the updated DaemonSet @@ -205,7 +213,6 @@ var _ = Describe("DaemonSet controller Suite", func() { clearReconciled() m.Update(daemonset, removeContainer2).Should(Succeed()) waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) // Get the updated DaemonSet m.Get(daemonset, timeout).Should(Succeed()) diff --git a/pkg/controller/daemonset/daemonset_webhook.go b/pkg/controller/daemonset/daemonset_webhook.go new file mode 100644 index 00000000..22605121 --- /dev/null +++ b/pkg/controller/daemonset/daemonset_webhook.go @@ -0,0 +1,39 @@ +package daemonset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-daemonset,mutating=true,failurePolicy=ignore,groups=apps,resources=daemonsets,verbs=create;update,versions=v1,name=daemonsets.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type DaemonSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDaemonSetWebhook(obj.(*appsv1.DaemonSet), request.DryRun) + return err +} + +func AddDaemonSetWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter( + &DaemonSetWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/controller/deployment/deployment_controller_suite_test.go b/pkg/controller/deployment/deployment_controller_suite_test.go index 0da71857..49c3213e 100644 --- a/pkg/controller/deployment/deployment_controller_suite_test.go +++ b/pkg/controller/deployment/deployment_controller_suite_test.go @@ -33,6 +33,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -47,8 +50,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-deployment" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "deployment-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "deployments.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"deployments"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/deployment/deployment_controller_test.go b/pkg/controller/deployment/deployment_controller_test.go index ba887726..5f394fc7 100644 --- a/pkg/controller/deployment/deployment_controller_test.go +++ b/pkg/controller/deployment/deployment_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("Deployment controller Suite", func() { @@ -101,6 +102,11 @@ var _ = Describe("Deployment controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -113,6 +119,10 @@ var _ = Describe("Deployment controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddDeploymentWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -188,8 +198,6 @@ var _ = Describe("Deployment controller Suite", func() { } clearReconciled() m.Update(deployment, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForDeploymentReconciled(deployment) waitForDeploymentReconciled(deployment) // Get the updated Deployment @@ -232,7 +240,6 @@ var _ = Describe("Deployment controller Suite", func() { clearReconciled() m.Update(deployment, removeContainer2).Should(Succeed()) waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) // Get the updated Deployment m.Get(deployment, timeout).Should(Succeed()) diff --git a/pkg/controller/deployment/deployment_webhook.go b/pkg/controller/deployment/deployment_webhook.go new file mode 100644 index 00000000..f75a126a --- /dev/null +++ b/pkg/controller/deployment/deployment_webhook.go @@ -0,0 +1,39 @@ +package deployment + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=ignore,groups=apps,resources=deployments,verbs=create;update,versions=v1,name=deployments.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type DeploymentWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DeploymentWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDeploymentWebhook(obj.(*appsv1.Deployment), request.DryRun) + return err +} + +func AddDeploymentWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter( + &DeploymentWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/controller/statefulset/statefulset_controller_suite_test.go b/pkg/controller/statefulset/statefulset_controller_suite_test.go index 93ddfd9f..67fc8705 100644 --- a/pkg/controller/statefulset/statefulset_controller_suite_test.go +++ b/pkg/controller/statefulset/statefulset_controller_suite_test.go @@ -33,6 +33,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -47,8 +50,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-statefulset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "statefulset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "statefulsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"statefulsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index 578aa0d2..28dbbf96 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("StatefulSet controller Suite", func() { @@ -95,6 +96,11 @@ var _ = Describe("StatefulSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) @@ -109,6 +115,10 @@ var _ = Describe("StatefulSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = AddStatefulSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -166,8 +176,6 @@ var _ = Describe("StatefulSet controller Suite", func() { } clearReconciled() m.Update(statefulset, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForStatefulSetReconciled(statefulset) waitForStatefulSetReconciled(statefulset) // Get the updated StatefulSet @@ -210,7 +218,6 @@ var _ = Describe("StatefulSet controller Suite", func() { clearReconciled() m.Update(statefulset, removeContainer2).Should(Succeed()) waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) // Get the updated StatefulSet m.Get(statefulset, timeout).Should(Succeed()) diff --git a/pkg/controller/statefulset/statefulset_webhook.go b/pkg/controller/statefulset/statefulset_webhook.go new file mode 100644 index 00000000..40a41a9e --- /dev/null +++ b/pkg/controller/statefulset/statefulset_webhook.go @@ -0,0 +1,39 @@ +package statefulset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-statefulset,mutating=true,failurePolicy=ignore,groups=apps,resources=statefulsets,verbs=create;update,versions=v1,name=statefulsets.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun + +type StatefulSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *StatefulSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleStatefulSetWebhook(obj.(*appsv1.StatefulSet), request.DryRun) + return err +} + +func AddStatefulSetWebhook(mgr manager.Manager) error { + err := builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter( + &StatefulSetWebhook{ + Client: mgr.GetClient(), + Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + }).Complete() + + return err +} diff --git a/pkg/core/children.go b/pkg/core/children.go index 9b109979..c4f957ef 100644 --- a/pkg/core/children.go +++ b/pkg/core/children.go @@ -22,6 +22,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,10 +39,19 @@ type configMetadata struct { keys map[string]struct{} } +type NotFoundError struct { + string +} + +func (e *NotFoundError) Error() string { + return e.string +} + // getResult is returned from the getObject method as a helper struct to be // passed into a channel type getResult struct { err error + notFound bool obj Object metadata configMetadata } @@ -68,11 +78,16 @@ func (h *Handler) getCurrentChildren(obj podController) ([]configObject, error) // Range over and collect results from the gets var errs []string + var notFoundErrs []string var children []configObject for i := 0; i < len(configMaps)+len(secrets); i++ { result := <-resultsChan if result.err != nil { - errs = append(errs, result.err.Error()) + if result.notFound { + notFoundErrs = append(notFoundErrs, result.err.Error()) + } else { + errs = append(errs, result.err.Error()) + } } if result.obj != nil { children = append(children, configObject{ @@ -89,6 +104,11 @@ func (h *Handler) getCurrentChildren(obj podController) ([]configObject, error) return []configObject{}, fmt.Errorf("error(s) encountered when geting children: %s", strings.Join(errs, ", ")) } + // If there we did not find required elements + if len(notFoundErrs) > 0 { + return []configObject{}, &NotFoundError{fmt.Sprintf("required children not found: %s", strings.Join(notFoundErrs, ", "))} + } + // No errors, return the list of children return children, nil } @@ -219,12 +239,16 @@ func (h *Handler) getObject(namespace, name string, metadata configMetadata, obj objectName := types.NamespacedName{Namespace: namespace, Name: name} err := h.Get(context.TODO(), objectName, obj) if err != nil { - if metadata.required { - return getResult{err: err} + if errors.IsNotFound(err) { + if metadata.required { + return getResult{err: err, notFound: true} + } + return getResult{metadata: metadata, notFound: true} + } else { + return getResult{err: err, notFound: false} } - return getResult{metadata: metadata} } - return getResult{obj: obj, metadata: metadata} + return getResult{obj: obj, metadata: metadata, notFound: false} } // getExistingChildren returns a list of all Secrets and ConfigMaps that are diff --git a/pkg/core/handler.go b/pkg/core/handler.go index 588eddf6..06940de9 100644 --- a/pkg/core/handler.go +++ b/pkg/core/handler.go @@ -48,6 +48,21 @@ func (h *Handler) HandleDeployment(instance *appsv1.Deployment) (reconcile.Resul return h.handlePodController(&deployment{Deployment: instance}) } +// HandleDeploymentWebhook is called by the deployment webhook +func (h *Handler) HandleDeploymentWebhook(instance *appsv1.Deployment, dryRun *bool) error { + return h.updatePodController(&deployment{Deployment: instance}, (dryRun != nil && *dryRun)) +} + +// HandleStatefulSetWebhook is called by the statefulset webhook +func (h *Handler) HandleStatefulSetWebhook(instance *appsv1.StatefulSet, dryRun *bool) error { + return h.updatePodController(&statefulset{StatefulSet: instance}, (dryRun != nil && *dryRun)) +} + +// HandleDaemonSetWebhook is called by the daemonset webhook +func (h *Handler) HandleDaemonSetWebhook(instance *appsv1.DaemonSet, dryRun *bool) error { + return h.updatePodController(&daemonset{DaemonSet: instance}, (dryRun != nil && *dryRun)) +} + // HandleStatefulSet is called by the StatefulSet controller to reconcile StatefulSets func (h *Handler) HandleStatefulSet(instance *appsv1.StatefulSet) (reconcile.Result, error) { return h.handlePodController(&statefulset{StatefulSet: instance}) @@ -74,6 +89,8 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, return reconcile.Result{}, nil } + log.V(5).Info("Reconciling") + // Get all children that the instance currently references current, err := h.getCurrentChildren(instance) if err != nil { @@ -104,3 +121,41 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, } return reconcile.Result{}, nil } + +// handlePodController will only update the hash. Everything else is left to the reconciler. +func (h *Handler) updatePodController(instance podController, dryRun bool) error { + log := logf.Log.WithName("wave").WithValues("namespace", instance.GetNamespace(), "name", instance.GetName(), "dryRun", dryRun) + log.V(5).Info("Running webhook") + + // If the required annotation isn't present, ignore the instance + if !hasRequiredAnnotation(instance) { + return nil + } + + // Get all children that the instance currently references + current, err := h.getCurrentChildren(instance) + if err != nil { + if _, ok := err.(*NotFoundError); ok { + log.V(0).Info("Not all required children found yet. Skipping mutation!", "err", err) + return nil + } else { + return fmt.Errorf("error fetching current children: %v", err) + } + } + + hash, err := calculateConfigHash(current) + if err != nil { + return fmt.Errorf("error calculating configuration hash: %v", err) + } + + // Update the desired state of the Deployment + oldHash := getConfigHash(instance) + setConfigHash(instance, hash) + + if !dryRun && oldHash != hash { + log.V(0).Info("Updating instance hash", "hash", oldHash) + h.recorder.Eventf(instance.GetApiObject(), corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) + } + + return nil +} diff --git a/pkg/core/hash.go b/pkg/core/hash.go index 85befda8..c179f73b 100644 --- a/pkg/core/hash.go +++ b/pkg/core/hash.go @@ -105,7 +105,7 @@ func getSecretData(child configObject) map[string][]byte { return keyData } -// setConfigHash upates the configuration hash of the given Deployment to the +// setConfigHash updates the configuration hash of the given Deployment to the // given string func setConfigHash(obj podController, hash string) { // Get the existing annotations @@ -120,3 +120,9 @@ func setConfigHash(obj podController, hash string) { podTemplate.SetAnnotations(annotations) obj.SetPodTemplate(podTemplate) } + +// getConfigHash return the config hash string +func getConfigHash(obj podController) string { + podTemplate := obj.GetPodTemplate() + return podTemplate.GetAnnotations()[ConfigHashAnnotation] +}