feat: proxy webhooks (#607)

* feat: add webhook handling and watchdog webhook Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com> * feat(helm): add webhook values and test Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com> * fix: disregard delete and generic events for watchdog Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com> * chore: align makefile$ Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com> --------- Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>
projectcapsule · Jan 14, 2025 · 69d8dda · 69d8dda
1 parent c338410
commit 69d8dda
Show file tree

Hide file tree

Showing 13 changed files with 475 additions and 35 deletions.
diff --git a/Makefile b/Makefile
@@ -106,19 +106,20 @@ helm-docs: docker
 helm-lint: docker
 	@docker run -v "$(SRC_ROOT):/workdir" --entrypoint /bin/sh quay.io/helmpack/chart-testing:v3.3.1 -c "cd /workdir; ct lint --config .github/configs/ct.yaml --lint-conf .github/configs/lintconf.yaml --all --debug"
 
-helm-test: helm-controller-version ct ko-build-all helm-create helm-install helm-destroy
+helm-test: helm-controller-version ct helm-create helm-install helm-destroy
 
-helm-install:
-	@kubectl apply --server-side=true -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml
-	@make install-capsule
-	@kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml
+helm-test-ct: helm-load-image
 	@$(CT) install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug
 
+helm-install: install-dependencies helm-test-ct
+
 helm-create: kind
 	@$(KIND) create cluster --wait=60s --name capsule-charts
-	@$(KIND) load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION)
 	@kubectl create ns capsule-system
 
+helm-load-image: kind helm-controller-version ko-build-all
+	@$(KIND) load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION)
+
 helm-destroy: kind
 	@$(KIND) delete cluster --name capsule-charts
 
@@ -142,15 +143,10 @@ e2e-build: kind
 	@echo "Building kubernetes env using Kind $${KIND_K8S_VERSION:-v1.27.0}..."
 	@$(KIND) create cluster --name capsule --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} --config ./e2e/kind.yaml --wait=120s \
 		&& kubectl taint nodes capsule-worker2 key1=value1:NoSchedule
-	@helm repo add bitnami https://charts.bitnami.com/bitnami
-	@helm repo update
-	@helm upgrade --install --namespace metrics-system --create-namespace metrics-server bitnami/metrics-server \
-		--set apiService.create=true --set "extraArgs[0]=--kubelet-insecure-tls=true" --version 6.2.9
 	@echo "Waiting for metrics-server pod to be ready for listing metrics"
-	@kubectl --namespace metrics-system wait --for=condition=ready --timeout=320s pod -l app.kubernetes.io/instance=metrics-server
 
 .PHONY: e2e-install
-e2e-install: install-capsule install-capsule-proxy rbac-fix
+e2e-install: install-capsule install-dependencies install-capsule-proxy rbac-fix
 
 .PHONY: e2e-load-image
 e2e-load-image: kind ko-build-all
@@ -184,12 +180,14 @@ ifeq ($(CAPSULE_PROXY_MODE),http)
 		--set "image.tag=$(VERSION)" \
 		--set "options.enableSSL=false" \
 		--set "options.logLevel=10" \
+		--set "options.pprof=true" \
 		--set "service.type=NodePort" \
 		--set "service.nodePort=" \
 		--set "kind=DaemonSet" \
 		--set "daemonset.hostNetwork=true" \
 		--set "serviceMonitor.enabled=false" \
 		--set "options.generateCertificates=false" \
+		--set "webhooks.enabled=true" \
 		--set "options.extraArgs={--feature-gates=ProxyClusterScoped=true,--feature-gates=ProxyAllNamespaced=true}"
 else
 	@echo "Running in HTTPS mode"
@@ -220,15 +218,28 @@ else
 		--set "image.pullPolicy=Never" \
 		--set "image.tag=$(VERSION)" \
 		--set "options.logLevel=10" \
+		--set "options.pprof=true" \
 		--set "service.type=NodePort" \
 		--set "service.nodePort=" \
 		--set "kind=DaemonSet" \
 		--set "daemonset.hostNetwork=true" \
 		--set "serviceMonitor.enabled=false" \
+		--set "webhooks.enabled=true" \
 		--set "options.extraArgs={--feature-gates=ProxyClusterScoped=true,--feature-gates=ProxyAllNamespaced=true}"
 endif
 	@kubectl rollout restart ds capsule-proxy -n capsule-system || true
 
+install-dependencies: install-capsule
+	@kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml
+	@helm repo add cert-manager https://charts.jetstack.io
+	@helm repo add bitnami https://charts.bitnami.com/bitnami
+	@helm repo update	
+	@helm upgrade --install cert-manager cert-manager/cert-manager --namespace cert-manager --create-namespace --version 1.16.2 --set crds.enabled=true 
+	@helm upgrade --install --namespace metrics-system --create-namespace metrics-server bitnami/metrics-server \
+		--set apiService.create=true --set "extraArgs[0]=--kubelet-insecure-tls=true" --version 6.2.9
+	@kubectl --namespace metrics-system wait --for=condition=ready --timeout=320s pod -l app.kubernetes.io/instance=metrics-server
+
+
 rbac-fix:
 	@echo "RBAC customization..."
 	@kubectl create clusterrole capsule-selfsubjectaccessreviews --verb=create --resource=selfsubjectaccessreviews.authorization.k8s.io

diff --git a/charts/capsule-proxy/README.md b/charts/capsule-proxy/README.md
@@ -104,7 +104,7 @@ If you only need to make minor customizations, you can specify them on the comma
 | global.jobs.certs.topologySpreadConstraints | list | `[]` | Set Topology Spread Constraints |
 | global.jobs.certs.ttlSecondsAfterFinished | int | `60` | Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. |
 | global.jobs.kubectl.affinity | object | `{}` | Set affinity rules |
-| global.jobs.kubectl.annotations | object | `{"helm.sh/hook-delete-policy":"before-hook-creation,hook-succeeded"}` | Annotations to add to the certgen job. |
+| global.jobs.kubectl.annotations | object | `{}` | Annotations |
 | global.jobs.kubectl.image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy of the helm chart job |
 | global.jobs.kubectl.image.registry | string | `"docker.io"` | Set the image repository of the helm chart job |
 | global.jobs.kubectl.image.repository | string | `"clastix/kubectl"` | Set the image repository of the helm chart job |
@@ -184,7 +184,9 @@ If you only need to make minor customizations, you can specify them on the comma
 | options.listeningPort | int | `9001` | Set the listening port of the capsule-proxy |
 | options.logLevel | string | `"4"` | Set the log verbosity of the capsule-proxy with a value from 1 to 10 |
 | options.oidcUsernameClaim | string | `"preferred_username"` | Specify if capsule-proxy will use SSL |
+| options.pprof | bool | `false` | Enable Pprof for profiling |
 | options.rolebindingsResyncPeriod | string | `"10h"` | Set the role bindings reflector resync period, a local cache to store mappings between users and their namespaces. [Use a lower value in case of flaky etcd server connections.](https://github.com/projectcapsule/capsule-proxy/issues/174) |
+| options.webhookPort | int | `9443` | Webhook port |
 
 ### Cert-Manager Parameters
 
@@ -203,6 +205,26 @@ You can manage the certificate with the help of [cert-manager](https://cert-mana
 | certManager.issuer.kind | string | `"Issuer"` | Set if the cert manager will generate either self-signed or CA signed SSL certificates. Its value will be either Issuer or ClusterIssuer |
 | certManager.issuer.name | string | `""` | Set the name of the ClusterIssuer if issuer kind is ClusterIssuer and if cert manager will generate CA signed SSL certificates |
 
+### Webhook Parameters
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| webhooks.certificate.dnsNames | list | `[]` | Additional DNS Names to include in certificate |
+| webhooks.certificate.fields | object | `{"privateKey":{"rotationPolicy":"Always"}}` | Additional fields to include in certificate |
+| webhooks.certificate.ipAddresses | list | `[]` | Additional IP Addresses to include in certificate |
+| webhooks.certificate.uris | list | `[]` | Additional URIs to include in certificate |
+| webhooks.enabled | bool | `false` | Enable the usage of mutating and validating webhooks |
+| webhooks.service.caBundle | string | `""` | CABundle for the webhook service |
+| webhooks.service.name | string | `""` | Custom service name for the webhook service |
+| webhooks.service.namespace | string | `""` | Custom service namespace for the webhook service |
+| webhooks.service.port | string | `nil` | Custom service port for the webhook service |
+| webhooks.service.url | string | `""` | The URL where the capsule webhook services are running (Overwrites cluster scoped service definition) |
+| webhooks.watchdog.enabled | bool | `true` | Enable Watchdog Webhook |
+| webhooks.watchdog.failurePolicy | string | `"Ignore"` | Ignore failures from the webhook |
+| webhooks.watchdog.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | Selects only namespaced items which are within a tenant |
+| webhooks.watchdog.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]` | Rules for which Objects and Actions this webhook should be called |
+| webhooks.watchdog.timeoutSeconds | string | `"3s"` | Timeout in seconds for mutating webhooks |
+
 ### Service Parameters
 
 | Key | Type | Default | Description |

diff --git a/charts/capsule-proxy/README.md.gotmpl b/charts/capsule-proxy/README.md.gotmpl
@@ -101,7 +101,7 @@ If you only need to make minor customizations, you can specify them on the comma
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 {{- range .Values }}
-  {{- if not (or (hasPrefix "certManager" .Key)  (hasPrefix "global" .Key) (hasPrefix "options" .Key) (hasPrefix "service." .Key) (hasPrefix "ingress" .Key) (hasPrefix "autoscaling" .Key) (hasPrefix "serviceMonitor" .Key) )  }}
+  {{- if not (or (hasPrefix "certManager" .Key) (hasPrefix "webhooks" .Key) (hasPrefix "global" .Key) (hasPrefix "options" .Key) (hasPrefix "service." .Key) (hasPrefix "ingress" .Key) (hasPrefix "autoscaling" .Key) (hasPrefix "serviceMonitor" .Key) )  }}
 | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} |
   {{- end }}
 {{- end }}
@@ -129,6 +129,16 @@ You can manage the certificate with the help of [cert-manager](https://cert-mana
   {{- end }}
 {{- end }}
 
+### Webhook Parameters
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+{{- range .Values }}
+  {{- if hasPrefix "webhooks." .Key }}
+| {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} |
+  {{- end }}
+{{- end }}
+
 
 ### Service Parameters
 

diff --git a/charts/capsule-proxy/ci/webhook-values.yaml b/charts/capsule-proxy/ci/webhook-values.yaml
@@ -0,0 +1,2 @@
+webhooks:
+  enabled: true
diff --git a/charts/capsule-proxy/templates/_helpers.tpl b/charts/capsule-proxy/templates/_helpers.tpl
@@ -100,3 +100,31 @@ Render the CLI flag --host values for the self-signed certificate generator
 {{- $values = append $values $fullname -}}
 {{ join "," $values }}
 {{- end -}}
+
+
+
+{{/*
+Capsule Webhook service (Called with $.Path)
+
+*/}}
+{{- define "capsule-proxy.webhooks.service" -}}
+  {{- include "capsule-proxy.webhooks.cabundle" $.ctx | nindent 0 }}
+  {{- if $.ctx.Values.webhooks.service.url }}
+url: {{ printf "%s/%s" (trimSuffix "/" $.ctx.Values.webhooks.service.url ) (trimPrefix "/" (required "Path is required for the function" $.path)) }}
+  {{- else }}
+service:
+  name: {{ default (printf "%s-webhook-service" (include "capsule-proxy.fullname" $.ctx)) $.ctx.Values.webhooks.service.name }}
+  namespace: {{ default $.ctx.Release.Namespace $.ctx.Values.webhooks.service.namespace }}
+  port: {{ default 443 $.ctx.Values.webhooks.service.port }}
+  path: {{ required "Path is required for the function" $.path }}
+  {{- end }}
+{{- end }}
+
+{{/*
+Capsule Webhook endpoint CA Bundle
+*/}}
+{{- define "capsule-proxy.webhooks.cabundle" -}}
+  {{- if $.Values.webhooks.service.caBundle -}}
+caBundle: {{ $.Values.webhooks.service.caBundle -}}
+  {{- end -}}
+{{- end -}}
diff --git a/charts/capsule-proxy/templates/_pod.tpl b/charts/capsule-proxy/templates/_pod.tpl
@@ -34,6 +34,13 @@ spec:
       secretName: {{ .Values.options.certificateVolumeName | default  (include "capsule-proxy.fullname" .) }}
       defaultMode: 420
   {{- end }}
+  {{- if .Values.webhooks.enabled }}
+  - name: webhook
+    secret:
+      secretName: {{ include "capsule-proxy.fullname" . }}-webhook-cert
+      defaultMode: 420
+  {{- end }}
+
   {{- with .Values.topologySpreadConstraints }}
   topologySpreadConstraints: {{- toYaml . | nindent 4 }}
   {{- end }}
@@ -45,6 +52,7 @@ spec:
     imagePullPolicy: {{ .Values.image.pullPolicy }}
     args:
     - --listening-port={{ .Values.options.listeningPort }}
+    - --webhook-port={{ .Values.options.webhookPort }}
     - --capsule-configuration-name={{ .Values.options.capsuleConfigurationName }}
     {{- range .Values.options.ignoredUserGroups }}
     - --ignored-user-group={{ . }}
@@ -61,6 +69,12 @@ spec:
     {{- end }}
     - --client-connection-qps={{ .Values.options.clientConnectionQPS }}
     - --client-connection-burst={{ .Values.options.clientConnectionBurst }}
+    - --enable-pprof={{ .Values.options.pprof }}
+    {{- if .Values.webhooks.enabled }}
+      {{- if .Values.webhooks.watchdog.enabled }}
+    - --webhooks=watchdog
+      {{- end }}
+    {{- end }}
     {{- with .Values.options.extraArgs }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
@@ -83,9 +97,16 @@ spec:
     - name: probe
       containerPort: 8081
       protocol: TCP
+    {{- if .Values.options.pprof }}
     - name: pprof
       containerPort: 8082
       protocol: TCP
+    {{- end }}
+    {{- if .Values.webhooks.enabled }}
+    - name: webhook
+      containerPort: {{ .Values.options.webhookPort }}
+      protocol: TCP
+    {{- end }}
     {{- if .Values.livenessProbe.enabled }}
     livenessProbe:
       {{- toYaml (omit .Values.livenessProbe "enabled") | nindent 6 }}
@@ -104,6 +125,11 @@ spec:
     - mountPath: {{ .Values.options.SSLDirectory }}
       name: certs
     {{- end }}
+    {{- if .Values.webhooks.enabled }}
+    - mountPath: /tmp/k8s-webhook-server/serving-certs
+      name: webhook
+      readOnly: true
+    {{- end }}
   {{- with .Values.nodeSelector }}
   nodeSelector:
     {{- toYaml . | nindent 8 }}

diff --git a/charts/capsule-proxy/templates/webhooks/certificate.yaml b/charts/capsule-proxy/templates/webhooks/certificate.yaml
@@ -0,0 +1,67 @@
+{{- if $.Values.webhooks.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook-ca
+spec:
+  isCA: true
+  commonName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
+  secretName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name:  {{ include "capsule-proxy.fullname" . }}-webhook-issuer
+    kind: Issuer
+    group: cert-manager.io
+---  
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook
+spec:
+  ca:
+    secretName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook-cert
+spec:
+  {{- with .Values.webhooks.certificate.fields }}
+    {{ toYaml . | nindent 2 }}
+  {{- end }}
+  dnsNames:
+  {{- range $dns := .Values.webhooks.certificate.dnsNames }}
+  - {{ $dns | quote }}
+  {{- end }}
+  - {{ include "capsule-proxy.fullname" . }}-webhook-service
+  - {{ include "capsule-proxy.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc
+  {{- with .Values.webhooks.certificate.ipAddresses }}
+  ipAddresses:
+    {{- range $ip := . }}
+  - {{ $ip }}
+    {{- end }}
+  {{- end }}
+  {{- with .Values.webhooks.certificate.uris }}
+  uris:
+    {{- range $uri := . }}
+  - {{ $uri }}
+    {{- end }}
+  {{- end }}
+  issuerRef:
+    kind: "Issuer"
+    name: {{ include "capsule-proxy.fullname" . }}-webhook
+  secretName: {{ include "capsule-proxy.fullname" . }}-webhook-cert
+  subject:
+    organizations:
+      - projectcapsule.dev
+{{- end }}
diff --git a/charts/capsule-proxy/templates/webhooks/mutating.yaml b/charts/capsule-proxy/templates/webhooks/mutating.yaml
@@ -0,0 +1,31 @@
+{{- if $.Values.webhooks.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook
+  labels:
+    {{- include "capsule-proxy.labels" . | nindent 4 }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "capsule-proxy.fullname" . }}-webhook-cert
+webhooks:
+  {{- with .Values.webhooks.watchdog }}
+    {{- if .enabled }}
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    {{- include "capsule-proxy.webhooks.service" (dict "path" "/mutate/watchdog" "ctx" $) | nindent 4 }}
+  failurePolicy: {{ .failurePolicy }}
+  name: watchdog.proxy.projectcapsule.dev
+  {{- with .rules }}
+  rules:
+    {{- toYaml .| nindent 4}} 
+  {{- end }}
+  {{- with .namespaceSelector }}
+  namespaceSelector:
+    {{- toYaml .| nindent 4}} 
+  {{- end }}
+  sideEffects: None
+  timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }}
+    {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/charts/capsule-proxy/templates/webhooks/service.yaml b/charts/capsule-proxy/templates/webhooks/service.yaml
@@ -0,0 +1,18 @@
+{{- if $.Values.webhooks.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "capsule-proxy.fullname" . }}-webhook-service
+  labels:
+    {{- include "capsule-proxy.labels" . | nindent 4 }}
+spec:
+  ports:
+  - port: 443
+    name: https
+    protocol: TCP
+    targetPort: {{ .Values.options.webhookPort }}
+  selector:
+    {{- include "capsule-proxy.selectorLabels" . | nindent 4 }}
+  sessionAffinity: None
+  type: ClusterIP
+{{- end }}