Add agent metrics annotations & env vars

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Unreleased
 
+## 0.16.0 (March 30, 2022)
+
+Features:
+* New environment variables and annotations to configure metrics on Vault agent [GH-329](https://github.com/hashicorp/vault-k8s/pull/329)
+
 ## 0.15.0 (March 21, 2022)
 
 Features:

agent-inject/agent/agent.go

@@ -33,6 +33,7 @@ const (
 	DefaultAgentInjectToken                 = false
 	DefaultTemplateConfigExitOnRetryFailure = true
 	DefaultServiceAccountMount              = "/var/run/secrets/vault.hashicorp.com/serviceaccount"
+	DefaultAgentMetricsListenerPort         = 8081
 )
 
 // Agent is the top level structure holding all the
@@ -158,6 +159,9 @@ type Agent struct {
 	// InjectToken controls whether the auto-auth token is injected into the
 	// secrets volume (e.g. /vault/secrets/token)
 	InjectToken bool
+
+	// VaultAgentMetricsListener is the structure holding the Vault agent metrics configuration
+	VaultAgentMetrics VaultAgentMetrics
 }
 
 type ServiceAccountTokenVolume struct {
@@ -291,6 +295,16 @@ type VaultAgentTemplateConfig struct {
 	StaticSecretRenderInterval string
 }
 
+type VaultAgentMetrics struct {
+	// Listener address enables and configures a listening address to
+	// receive http metrics requests
+	ListenerAddress string
+
+	// PrometheusRetention enables prometheus metrics and
+	// configures the retention time
+	PrometheusRetention string
+}
+
 // New creates a new instance of Agent by parsing all the Kubernetes annotations.
 func New(pod *corev1.Pod, patches []*jsonpatch.JsonPatchOperation) (*Agent, error) {
 	sa, err := serviceaccount(pod)
@@ -441,6 +455,12 @@ func New(pod *corev1.Pod, patches []*jsonpatch.JsonPatchOperation) (*Agent, erro
 		StaticSecretRenderInterval: pod.Annotations[AnnotationTemplateConfigStaticSecretRenderInterval],
 	}
 
+	agent.VaultAgentMetrics.ListenerAddress, err = agent.metricsListenerAddress()
+	if err != nil {
+		return agent, err
+	}
+	agent.VaultAgentMetrics.PrometheusRetention = pod.Annotations[AnnotationAgentMetricsPrometheusRetention]
+
 	return agent, nil
 }
 

agent-inject/agent/annotations.go

@@ -255,6 +255,14 @@ const (
 	// If specified, configures how often Vault Agent Template should render non-leased secrets such as KV v2.
 	// Defaults to 5 minutes.
 	AnnotationTemplateConfigStaticSecretRenderInterval = "vault.hashicorp.com/template-static-secret-render-interval"
+
+	// AnnotationAgentMetricsListenerAddress
+	// If specified, configures a TCP listener on the Vault Agent to receive metrics requests from
+	AnnotationAgentMetricsListenerAddress = "vault.hashicorp.com/agent-metrics-listener-address"
+
+	// AnnotationAgentMetricsPrometheusRetention
+	// If specified, enables prometheus metrics and configures the amount of time they are retained in memory
+	AnnotationAgentMetricsPrometheusRetention = "vault.hashicorp.com/agent-metrics-prometheus-retention"
 )
 
 type AgentConfig struct {
@@ -276,6 +284,8 @@ type AgentConfig struct {
 	ResourceLimitMem           string
 	ExitOnRetryFailure         bool
 	StaticSecretRenderInterval string
+	MetricsListenerAddress     string
+	MetricsPrometheusRetention string
 }
 
 // Init configures the expected annotations required to create a new instance
@@ -443,6 +453,14 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error {
 		pod.ObjectMeta.Annotations[AnnotationTemplateConfigStaticSecretRenderInterval] = cfg.StaticSecretRenderInterval
 	}
 
+	if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentMetricsListenerAddress]; !ok {
+		pod.ObjectMeta.Annotations[AnnotationAgentMetricsListenerAddress] = cfg.MetricsListenerAddress
+	}
+
+	if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentMetricsPrometheusRetention]; !ok {
+		pod.ObjectMeta.Annotations[AnnotationAgentMetricsPrometheusRetention] = cfg.MetricsPrometheusRetention
+	}
+
 	return nil
 }
 
@@ -689,3 +707,36 @@ func (a *Agent) authConfig() map[string]interface{} {
 
 	return authConfig
 }
+
+func (a *Agent) metricsListenerAddress() (string, error) {
+	addr := a.Annotations[AnnotationAgentMetricsListenerAddress]
+	if addr == "" {
+		return "", nil
+	}
+
+	arr := strings.Split(addr, ":")
+	switch len(arr) {
+	case 1:
+		addr = fmt.Sprintf("%s:%d", addr, DefaultAgentMetricsListenerPort)
+	case 2:
+	default:
+		return "", fmt.Errorf("invalid metrics address string: %s", addr)
+	}
+
+	// Temporary safeguard until the issue described below is resolved.
+	// In many cases, users will intend for the cache listener to only be available
+	// locally while exposing the metrics endpoint externally. At the time of writing
+	// this though, the "vault agent -config" command, which the Agent Injector relies
+	// on, will configure each listener to expose both the cache and metrics endpoints,
+	// regardless of the listener's intended purpose. This can be particularly dangerous
+	// if the cache has been configured to auto-authenticate.
+	// See https://github.com/remilapeyre/vault/blob/1e331ed2973b1099e3dfc169ced2d3c2d0364fc0/command/agent.go#L723
+	if cacheEnabled, err := strconv.ParseBool(a.Annotations[AnnotationAgentCacheEnable]); err == nil {
+		autoAuth := a.Annotations[AnnotationAgentCacheUseAutoAuthToken]
+		if cacheEnabled && (arr[0] != "127.0.0.1") && (autoAuth != "false") && (autoAuth != "") {
+			return "", fmt.Errorf("metrics listener address must be 127.0.0.1 if the agent cache is enabled.")
+		}
+	}
+
+	return addr, nil
+}

agent-inject/agent/annotations_test.go

@@ -732,6 +732,10 @@ func TestCouldErrorAnnotations(t *testing.T) {
 		{AnnotationAgentCacheEnable, "tRuE", false},
 		{AnnotationAgentCacheEnable, "fAlSe", false},
 		{AnnotationAgentCacheEnable, "", false},
+
+		{AnnotationAgentMetricsListenerAddress, "127.0.0.1", true},
+		{AnnotationAgentMetricsListenerAddress, "127.0.0.1:8080", true},
+		{AnnotationAgentMetricsListenerAddress, "127.0.01:1:1", false},
 	}
 
 	for i, tt := range tests {

agent-inject/agent/config.go

@@ -27,6 +27,7 @@ type Config struct {
 	Listener       []*Listener     `json:"listener,omitempty"`
 	Cache          *Cache          `json:"cache,omitempty"`
 	TemplateConfig *TemplateConfig `json:"template_config,omitempty"`
+	Telemetry      *Telemetry      `json:"telemetry,omitempty"`
 }
 
 // Vault contains configuration for connecting to Vault servers
@@ -80,7 +81,7 @@ type Template struct {
 	Perms          string `json:"perms,omitempty"`
 }
 
-// Listener defines the configuration for Vault Agent Cache Listener
+// Listener defines the configuration for Vault Agent Cache Listener and/ or Vault Agent Metrics Listener
 type Listener struct {
 	Type       string `json:"type"`
 	Address    string `json:"address"`
@@ -108,6 +109,12 @@ type TemplateConfig struct {
 	StaticSecretRenderInterval string `json:"static_secret_render_interval,omitempty"`
 }
 
+// Telemetry defines the telemetry configuration for Vault Agent
+type Telemetry struct {
+	PrometheusRetentionTime string `json:"prometheus_retention_time"`
+	DisableHostname         bool   `json:"disable_hostname"`
+}
+
 func (a *Agent) newTemplateConfigs() []*Template {
 	var templates []*Template
 	for _, secret := range a.Secrets {
@@ -215,6 +222,25 @@ func (a *Agent) newConfig(init bool) ([]byte, error) {
 		}
 	}
 
+	if a.VaultAgentMetrics.ListenerAddress != "" && !init {
+		config.Listener = append(config.Listener, &Listener{
+			Type:       "tcp",
+			Address:    a.VaultAgentMetrics.ListenerAddress,
+			TLSDisable: true,
+		})
+
+		if config.Cache == nil { // workaround until https://github.com/hashicorp/vault/issues/8953 is resolved
+			config.Cache = &Cache{}
+		}
+	}
+
+	if a.VaultAgentMetrics.PrometheusRetention != "" && !init {
+		config.Telemetry = &Telemetry{
+			PrometheusRetentionTime: a.VaultAgentMetrics.PrometheusRetention,
+			DisableHostname:         true,
+		}
+	}
+
 	return config.render()
 }
 

agent-inject/handler.go

@@ -64,6 +64,8 @@ type Handler struct {
 	ResourceLimitMem           string
 	ExitOnRetryFailure         bool
 	StaticSecretRenderInterval string
+	MetricsListenerAddress     string
+	MetricsPrometheusRetention string
 }
 
 // Handle is the http.HandlerFunc implementation that actually handles the
@@ -194,6 +196,8 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi
 		ResourceLimitMem:           h.ResourceLimitMem,
 		ExitOnRetryFailure:         h.ExitOnRetryFailure,
 		StaticSecretRenderInterval: h.StaticSecretRenderInterval,
+		MetricsListenerAddress:     h.MetricsListenerAddress,
+		MetricsPrometheusRetention: h.MetricsPrometheusRetention,
 	}
 	err = agent.Init(&pod, cfg)
 	if err != nil {

subcommand/injector/command.go

@@ -56,7 +56,7 @@ type Command struct {
 	flagRunAsGroup                 string // Group (gid) to run Vault agent as
 	flagRunAsSameUser              bool   // Run Vault agent as the User (uid) of the first application container
 	flagSetSecurityContext         bool   // Set SecurityContext in injected containers
-	flagTelemetryPath              string // Path under which to expose metrics
+	flagTelemetryPath              string // Path under which to expose Agent Injector metrics
 	flagUseLeaderElector           bool   // Use leader elector code
 	flagDefaultTemplate            string // Toggles which default template to use
 	flagResourceRequestCPU         string // Set CPU request in the injected containers
@@ -65,6 +65,8 @@ type Command struct {
 	flagResourceLimitMem           string // Set Memory limit in the injected containers
 	flagTLSMinVersion              string // Minimum TLS version supported by the webhook server
 	flagTLSCipherSuites            string // Comma-separated list of supported cipher suites
+	flagMetricsListenerAddress     string // Enables and defines a default listener address for Vault agent
+	flagMetricsPrometheusRetention string // Enables prometheus metrics and defines a default retention period for Vault agent
 
 	flagSet *flag.FlagSet
 
@@ -197,6 +199,8 @@ func (c *Command) Run(args []string) int {
 		ResourceLimitMem:           c.flagResourceLimitMem,
 		ExitOnRetryFailure:         c.flagExitOnRetryFailure,
 		StaticSecretRenderInterval: c.flagStaticSecretRenderInterval,
+		MetricsListenerAddress:     c.flagMetricsListenerAddress,
+		MetricsPrometheusRetention: c.flagMetricsPrometheusRetention,
 	}
 
 	mux := http.NewServeMux()

subcommand/injector/flags.go

@@ -109,6 +109,12 @@ type Specification struct {
 
 	// TLSCipherSuites is the AGENT_INJECT_TLS_CIPHER_SUITES environment variable
 	TLSCipherSuites string `envconfig:"tls_cipher_suites"`
+
+	// MetricsListenerAddress is the AGENT_INJECT_METRICS_LISTENER_ADDR environment variable
+	MetricsListenerAddr string `split_words:"true"`
+
+	// MetricsPrometheusRetention is the AGENT_INJECT_METRICS_PROMETHEUS_RETENTION environment variable
+	MetricsPrometheusRetention string `split_words:"true"`
 }
 
 func (c *Command) init() {
@@ -153,7 +159,7 @@ func (c *Command) init() {
 	c.flagSet.BoolVar(&c.flagSetSecurityContext, "set-security-context", agent.DefaultAgentSetSecurityContext,
 		fmt.Sprintf("Set SecurityContext in injected containers. Defaults to %v.", agent.DefaultAgentSetSecurityContext))
 	c.flagSet.StringVar(&c.flagTelemetryPath, "telemetry-path", "",
-		"Path under which to expose metrics")
+		"Path under which to expose Agent Injector metrics")
 	c.flagSet.BoolVar(&c.flagUseLeaderElector, "use-leader-elector", agent.DefaultAgentUseLeaderElector,
 		"Use leader elector to coordinate multiple replicas when updating CA and Certs with auto-tls")
 	c.flagSet.StringVar(&c.flagDefaultTemplate, "default-template", agent.DefaultTemplateType,
@@ -180,6 +186,11 @@ func (c *Command) init() {
 	c.flagSet.StringVar(&c.flagTLSCipherSuites, "tls-cipher-suites", "",
 		"Comma-separated list of supported cipher suites for TLS 1.0-1.2")
 
+	c.flagSet.StringVar(&c.flagMetricsListenerAddress, "metrics-listener-address", "",
+		"Enables and defines a default listener address for Vault agent")
+	c.flagSet.StringVar(&c.flagMetricsPrometheusRetention, "metrics-prometheus-retention", "",
+		"Enables prometheus metrics and defines a default retention period for Vault agent")
+
 	c.help = flags.Usage(help, c.flagSet)
 }
 
@@ -339,5 +350,13 @@ func (c *Command) parseEnvs() error {
 		c.flagTLSCipherSuites = envs.TLSCipherSuites
 	}
 
+	if envs.MetricsListenerAddr != "" {
+		c.flagMetricsListenerAddress = envs.MetricsListenerAddr
+	}
+
+	if envs.MetricsPrometheusRetention != "" {
+		c.flagMetricsPrometheusRetention = envs.MetricsPrometheusRetention
+	}
+
 	return nil
 }