From f3f6b3a4993881a543bb7b771f9a827895387429 Mon Sep 17 00:00:00 2001 From: Guilherme Santos <157053549+gsantos-hc@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:20:07 -0500 Subject: [PATCH] feat: add agent injector telemetry Add Prometheus metrics to monitor the Agent Injector's performance. New metrics include a gauge of current requests being processed by the webhook, a summary of request processing times, and a count of successful and failed injections by Kubernetes namespace. Successful injections are broken down by injection type. The `injection_type` label can assume the value `init_only` for injections with only an initContainer (no sidecar) and `sidecar` for all other cases (sidecar only or sidecar + initContainer). Fixes AG-005161. --- agent-inject/handler.go | 27 +++++++++++++ agent-inject/metrics.go | 73 ++++++++++++++++++++++++++++++++++ subcommand/injector/command.go | 2 + 3 files changed, 102 insertions(+) create mode 100644 agent-inject/metrics.go diff --git a/agent-inject/handler.go b/agent-inject/handler.go index 4eed7047..05e00060 100644 --- a/agent-inject/handler.go +++ b/agent-inject/handler.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "strings" + "time" "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault-k8s/agent-inject/agent" @@ -25,6 +26,8 @@ import ( "k8s.io/client-go/kubernetes" ) +const warningInitOnlyInjection = "workload uses initContainer only; consider migrating to VSO" + var ( admissionScheme = runtime.NewScheme() deserializer = func() runtime.Decoder { @@ -85,6 +88,14 @@ type Handler struct { func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) { h.Log.Info("Request received", "Method", r.Method, "URL", r.URL) + // Measure request processing duration and monitor request queue + requestQueue.Inc() + requestStart := time.Now() + defer func() { + requestProcessingTime.Observe(float64(time.Since(requestStart).Milliseconds())) + requestQueue.Dec() + }() + if ct := r.Header.Get("Content-Type"); ct != "application/json" { msg := fmt.Sprintf("Invalid content-type: %q", ct) http.Error(w, msg, http.StatusBadRequest) @@ -142,11 +153,20 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) { msg := fmt.Sprintf("error marshalling admission response: %s", err) http.Error(w, msg, http.StatusInternalServerError) h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError) + incrementInjectionFailures(admReq.Request.Namespace) return } if _, err := w.Write(resp); err != nil { h.Log.Error("error writing response", "Error", err) + incrementInjectionFailures(admReq.Request.Namespace) + return + } + + if admResp.Response.Allowed { + incrementInjections(admReq.Request.Namespace, *admResp.Response) + } else { + incrementInjectionFailures(admReq.Request.Namespace) } } @@ -245,6 +265,13 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi return admissionError(req.UID, err) } + // Expose when workloads are injecting initContainers only. This is useful to identify + // workloads that are likely good candidates for migration to Vault Secrets Operator for + // greater scalability and performance. + if agentSidecar.PrePopulateOnly { + resp.Warnings = append(resp.Warnings, warningInitOnlyInjection) + } + resp.Patch = patch patchType := admissionv1.PatchTypeJSONPatch resp.PatchType = &patchType diff --git a/agent-inject/metrics.go b/agent-inject/metrics.go new file mode 100644 index 00000000..6efcf0f4 --- /dev/null +++ b/agent-inject/metrics.go @@ -0,0 +1,73 @@ +package agent_inject + +import ( + "slices" + + "github.com/prometheus/client_golang/prometheus" + admissionv1 "k8s.io/api/admission/v1" +) + +const ( + metricsNamespace = "vault" + metricsSubsystem = "agent_injector" + metricsLabelNamespace = "namespace" + metricsLabelType = "injection_type" + metricsLabelTypeDefault = "sidecar" + metricsLabelTypeInitOnly = "init_only" +) + +var ( + requestQueue = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "request_queue_total", + Help: "Total count of webhook requests in the queue", + }) + + requestProcessingTime = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "request_processing_duration_ms", + Help: "Histogram of webhook request processing times in milliseconds", + Buckets: []float64{5, 10, 25, 50, 75, 100, 250, 500, 1000, 2500, 5000, 7500, 10000}, + }) + + injectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "injections_by_namespace_total", + Help: "Total count of Agent Sidecar injections by namespace", + }, []string{metricsLabelNamespace, metricsLabelType}) + + failedInjectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "failed_injections_by_namespace_total", + Help: "Total count of failed Agent Sidecar injections by namespace", + }, []string{metricsLabelNamespace}) +) + +func incrementInjections(namespace string, res admissionv1.AdmissionResponse) { + typeLabel := metricsLabelTypeDefault + if slices.Contains(res.Warnings, warningInitOnlyInjection) { + typeLabel = metricsLabelTypeInitOnly + } + + injectionsByNamespace.With(prometheus.Labels{ + metricsLabelNamespace: namespace, + metricsLabelType: typeLabel, + }).Inc() +} + +func incrementInjectionFailures(namespace string) { + failedInjectionsByNamespace.With(prometheus.Labels{metricsLabelNamespace: namespace}).Inc() +} + +func MustRegisterInjectorMetrics(registry prometheus.Registerer) { + registry.MustRegister( + requestQueue, + requestProcessingTime, + injectionsByNamespace, + failedInjectionsByNamespace, + ) +} diff --git a/subcommand/injector/command.go b/subcommand/injector/command.go index 94e27e8d..75bcd0a5 100644 --- a/subcommand/injector/command.go +++ b/subcommand/injector/command.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/vault-k8s/leader" "github.com/hashicorp/vault-k8s/version" "github.com/mitchellh/cli" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" adminv1 "k8s.io/api/admissionregistration/v1" adminv1beta "k8s.io/api/admissionregistration/v1beta1" @@ -231,6 +232,7 @@ func (c *Command) Run(args []string) int { // Registering path to expose metrics if c.flagTelemetryPath != "" { + agentInject.MustRegisterInjectorMetrics(prometheus.DefaultRegisterer) c.UI.Info(fmt.Sprintf("Registering telemetry path on %q", c.flagTelemetryPath)) mux.Handle(c.flagTelemetryPath, promhttp.Handler()) }