Create a mutating webhook to mount secrets

CHANGELOG/CHANGELOG-1.6.md

@@ -24,3 +24,4 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 * [BUGFIX] [#854](https://github.com/k8ssandra/k8ssandra-operator/issues/854) Use Patch() to update K8ssandraTask status.
 * [CHANGE] [#887](https://github.com/k8ssandra/k8ssandra-operator/issues/887) Fix CVE-2022-32149.
 * [CHANGE] [#891](https://github.com/k8ssandra/k8ssandra-operator/issues/848) Update golang.org/x/net to fix several CVEs.
+* [FEATURE] [#605](https://github.com/k8ssandra/k8ssandra-operator/issues/598) Create a mutating webhook for the internal secrets provider

config/components/single-namespace/kustomization.yaml

@@ -27,6 +27,11 @@ replacements:
         kind: ValidatingWebhookConfiguration
       fieldPaths:
       - webhooks.0.clientConfig.service.namespace
+    - select:
+        name: k8ssandra-operator-mutating-webhook-configuration
+        kind: MutatingWebhookConfiguration
+      fieldPaths:
+      - webhooks.0.clientConfig.service.namespace
 patchesStrategicMerge:
 - |-
   apiVersion: v1

config/default/webhookcainjection_patch.yaml

@@ -1,12 +1,12 @@
 # This patch add annotation to admission webhook config and
 # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
-# apiVersion: admissionregistration.k8s.io/v1
-# kind: MutatingWebhookConfiguration
-# metadata:
-#   name: mutating-webhook-configuration
-#   annotations:
-#     cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-# ---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: k8ssandra-operator-mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE_K8S)/$(CERTIFICATE_NAME_K8S)
+---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:

config/webhook/kustomization.yaml

@@ -20,3 +20,24 @@ patchesJson6902:
     - op: replace
       path: /webhooks/0/clientConfig/service/name
       value: k8ssandra-operator-webhook-service
+
+# adding the objectSelector prevents the bootstrapping problem
+# where the mutation request for the operator pod would be 
+# sent before the operator pod is created
+patchesJson6902:
+- target:
+    group: admissionregistration.k8s.io
+    version: v1
+    name: k8ssandra-operator-mutating-webhook-configuration
+    kind: MutatingWebhookConfiguration
+  patch: |-
+    - op: replace
+      path: /webhooks/0/clientConfig/service/name
+      value: k8ssandra-operator-webhook-service
+    - op: add
+      path: /webhooks/0/objectSelector
+      value:
+        matchExpressions:
+          - key: control-plane
+            operator: NotIn
+            values: ["k8ssandra-operator"]

config/webhook/kustomizeconfig.yaml

@@ -4,18 +4,18 @@ nameReference:
 - kind: Service
   version: v1
   fieldSpecs:
-  # - kind: MutatingWebhookConfiguration
-  #   group: admissionregistration.k8s.io
-  #   path: webhooks/clientConfig/service/name
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
   - kind: ValidatingWebhookConfiguration
     group: admissionregistration.k8s.io
     path: webhooks/clientConfig/service/name
 
 namespace:
-# - kind: MutatingWebhookConfiguration
-#   group: admissionregistration.k8s.io
-#   path: webhooks/clientConfig/service/namespace
-#   create: true
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
 - kind: ValidatingWebhookConfiguration
   group: admissionregistration.k8s.io
   path: webhooks/clientConfig/service/namespace

config/webhook/manifests.yaml

@@ -1,5 +1,32 @@
 ---
 apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-v1-pod-secrets-inject
+  failurePolicy: Fail
+  name: mpod.kb.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
   creationTimestamp: null

controllers/medusa/controllers_test.go

@@ -9,6 +9,7 @@ import (
 
 	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
 	k8ssandractrl "github.com/k8ssandra/k8ssandra-operator/controllers/k8ssandra"
+	secretswebhook "github.com/k8ssandra/k8ssandra-operator/controllers/secrets-webhook"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/clientcache"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/config"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -81,7 +82,15 @@ func setupMedusaBackupTestEnv(t *testing.T, ctx context.Context) *testutils.Mult
 		}
 
 		for _, env := range testEnv.GetDataPlaneEnvTests() {
-			dataPlaneMgr, err := ctrl.NewManager(env.Config, ctrl.Options{Scheme: scheme.Scheme})
+			dataPlaneMgr, err := ctrl.NewManager(
+				env.Config,
+				ctrl.Options{
+					Scheme:  scheme.Scheme,
+					Host:    env.WebhookInstallOptions.LocalServingHost,
+					Port:    env.WebhookInstallOptions.LocalServingPort,
+					CertDir: env.WebhookInstallOptions.LocalServingCertDir,
+				},
+			)
 			if err != nil {
 				return err
 			}
@@ -94,6 +103,8 @@ func setupMedusaBackupTestEnv(t *testing.T, ctx context.Context) *testutils.Mult
 			if err != nil {
 				return err
 			}
+			secretswebhook.SetupSecretsInjectorWebhook(dataPlaneMgr)
+
 			go func() {
 				err := dataPlaneMgr.Start(ctx)
 				if err != nil {
@@ -214,7 +225,15 @@ func setupMedusaTaskTestEnv(t *testing.T, ctx context.Context) *testutils.MultiC
 		}
 
 		for _, env := range testEnv.GetDataPlaneEnvTests() {
-			dataPlaneMgr, err := ctrl.NewManager(env.Config, ctrl.Options{Scheme: scheme.Scheme})
+			dataPlaneMgr, err := ctrl.NewManager(
+				env.Config,
+				ctrl.Options{
+					Scheme:  scheme.Scheme,
+					Host:    env.WebhookInstallOptions.LocalServingHost,
+					Port:    env.WebhookInstallOptions.LocalServingPort,
+					CertDir: env.WebhookInstallOptions.LocalServingCertDir,
+				},
+			)
 			if err != nil {
 				return err
 			}
@@ -233,6 +252,8 @@ func setupMedusaTaskTestEnv(t *testing.T, ctx context.Context) *testutils.MultiC
 				Scheme:           scheme.Scheme,
 				ClientFactory:    medusaClientFactory,
 			}).SetupWithManager(dataPlaneMgr)
+			secretswebhook.SetupSecretsInjectorWebhook(dataPlaneMgr)
+
 			if err != nil {
 				return err
 			}

controllers/secrets-webhook/secretswebhook.go

@@ -0,0 +1,151 @@
+package secrets_webhook
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	"net/http"
+
+	"github.com/k8ssandra/k8ssandra-operator/pkg/utils"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	log "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// +kubebuilder:webhook:path=/mutate-v1-pod-secrets-inject,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1,sideEffects=None
+
+func SetupSecretsInjectorWebhook(mgr ctrl.Manager) {
+	mgr.GetWebhookServer().Register("/mutate-v1-pod-secrets-inject", &webhook.Admission{Handler: &podSecretsInjector{Client: mgr.GetClient()}})
+}
+
+// podSecretsInjector is an admission handler that mutates pod manifests
+// to include mechanisms for mounting secrets
+type podSecretsInjector struct {
+	Client  client.Client
+	decoder *admission.Decoder
+}
+
+// podSecretsInjector Implements admission.Handler.
+var _ admission.Handler = &podSecretsInjector{}
+
+// InjectDecoder injects the decoder into the podSecretsInjector
+func (p *podSecretsInjector) InjectDecoder(d *admission.Decoder) error {
+	p.decoder = d
+	return nil
+}
+
+func (p *podSecretsInjector) Handle(ctx context.Context, req admission.Request) admission.Response {
+	logger := log.FromContext(ctx).WithValues("podSecretsInjector", req.Namespace)
+
+	pod := &corev1.Pod{}
+	err := p.decoder.Decode(req, pod)
+	if err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	copy := pod.DeepCopy()
+
+	err = p.mutatePods(ctx, copy, logger)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	marshaledPod, err := json.Marshal(copy)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
+}
+
+// e.g. k8ssandra.io/inject-secret: '[{ "secretName": "test-secret", "path": "/etc/test/test-secret" }]'
+const secretInjectionAnnotation = "k8ssandra.io/inject-secret"
+
+type SecretInjection struct {
+	SecretName string `json:"secretName"`
+	Path       string `json:"path"`
+}
+
+// mutatePods injects the secret mounting configuration into the pod
+func (p *podSecretsInjector) mutatePods(ctx context.Context, pod *corev1.Pod, logger logr.Logger) error {
+	if pod.Annotations == nil {
+		logger.Info("no annotations exist", "podName", pod.Name, "namespace", pod.Namespace)
+		return nil
+	}
+
+	secretsStr := pod.Annotations[secretInjectionAnnotation]
+	if len(secretsStr) == 0 {
+		logger.Info("no secret annotation exists", "podName", pod.Name, "namespace", pod.Namespace)
+		return nil
+	}
+
+	var secrets []SecretInjection
+	if err := json.Unmarshal([]byte(secretsStr), &secrets); err != nil {
+		logger.Error(err, "unable to unmarhsal secrets annotation",
+			"annotation", secretsStr,
+			"podName", pod.Name,
+			"namespace", pod.Namespace,
+		)
+		return err
+	}
+
+	for _, secret := range secrets {
+		// get secret name from injection annotation
+		secretName := secret.SecretName
+		mountPath := secret.Path
+		logger.Info("creating volume and volume mount for secret",
+			"secret", secretName,
+			"secret path", mountPath,
+			"podName", pod.Name,
+			"namespace", pod.Namespace,
+		)
+
+		volume := corev1.Volume{
+			Name: secretName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		injectVolume(pod, volume)
+
+		volumeMount := corev1.VolumeMount{
+			Name:      secretName,
+			MountPath: mountPath,
+		}
+		injectVolumeMount(pod, volumeMount)
+		logger.Info("added volume and volumeMount to podSpec",
+			"secret", secretName,
+			"secret path", mountPath,
+			"podName", pod.Name,
+			"namespace", pod.Namespace,
+		)
+	}
+
+	return nil
+}
+
+// injectVolume attaches a volume to the pod spec
+func injectVolume(pod *corev1.Pod, volume corev1.Volume) {
+	if _, found := utils.ContainsVolume(pod.Spec.Volumes, volume.Name); !found {
+		pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+	}
+}
+
+// injectVolumeMount attaches a volumeMount to all containers in the pod spec
+func injectVolumeMount(pod *corev1.Pod, volumeMount corev1.VolumeMount) {
+	for i, container := range pod.Spec.Containers {
+		if utils.FindVolumeMount(&container, volumeMount.Name) == nil {
+			pod.Spec.Containers[i].VolumeMounts = append(container.VolumeMounts, volumeMount)
+		}
+	}
+	for i, container := range pod.Spec.InitContainers {
+		if utils.FindVolumeMount(&container, volumeMount.Name) == nil {
+			pod.Spec.Containers[i].VolumeMounts = append(container.VolumeMounts, volumeMount)
+		}
+	}
+}