From 077924bd65e3a9c2cbfd36e0f60b10826809053f Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Mon, 28 Nov 2022 16:17:41 -0500 Subject: [PATCH 01/10] VAULT-11510 Vault Agent can start listeners without caching --- command/agent.go | 174 ++++++++++-------- command/agent/cache/api_proxy_test.go | 36 ++++ command/agent/cache/cache_test.go | 65 ++++--- command/agent/cache/handler.go | 60 +++++- command/agent/cache/lease_cache.go | 5 + command/agent/cache_end_to_end_test.go | 2 +- command/agent/config/config.go | 12 +- command/agent/config/config_test.go | 126 +++++++++++-- .../config-disable-idle-connections-all.hcl | 2 +- ...nfig-disable-idle-connections-proxying.hcl | 27 +++ .../config-disable-keep-alives-all.hcl | 2 +- .../config-disable-keep-alives-proxying.hcl | 27 +++ website/content/docs/agent/caching/index.mdx | 10 +- website/content/docs/agent/index.mdx | 26 ++- 14 files changed, 439 insertions(+), 135 deletions(-) create mode 100644 command/agent/config/test-fixtures/config-disable-idle-connections-proxying.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-proxying.hcl diff --git a/command/agent.go b/command/agent.go index 91921fc9946d..5a83db3467d6 100644 --- a/command/agent.go +++ b/command/agent.go @@ -16,6 +16,9 @@ import ( "sync" "time" + "github.com/hashicorp/vault/internalshared/listenerutil" + "google.golang.org/grpc/test/bufconn" + systemd "github.com/coreos/go-systemd/daemon" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/gatedwriter" @@ -44,7 +47,6 @@ import ( "github.com/hashicorp/vault/helper/logging" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/internalshared/configutil" - "github.com/hashicorp/vault/internalshared/listenerutil" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/useragent" "github.com/hashicorp/vault/sdk/logical" @@ -53,7 +55,6 @@ import ( "github.com/mitchellh/cli" "github.com/oklog/run" "github.com/posener/complete" - "google.golang.org/grpc/test/bufconn" ) var ( @@ -463,23 +464,25 @@ func (c *AgentCommand) Run(args []string) int { var leaseCache *cache.LeaseCache var previousToken string - // Parse agent listener configurations - if config.Cache != nil { - cacheLogger := c.logger.Named("cache") + var cacheHandler http.Handler - proxyClient, err := client.CloneWithHeaders() - if err != nil { - c.UI.Error(fmt.Sprintf("Error cloning client for caching: %v", err)) - return 1 - } + proxyClient, err := client.CloneWithHeaders() + if err != nil { + c.UI.Error(fmt.Sprintf("Error cloning client for proxying: %v", err)) + return 1 + } - if config.DisableIdleConnsCaching { - proxyClient.SetMaxIdleConnections(-1) - } + if config.DisableIdleConnsAPIProxy { + proxyClient.SetMaxIdleConnections(-1) + } - if config.DisableKeepAlivesCaching { - proxyClient.SetDisableKeepAlives(true) - } + if config.DisableKeepAlivesAPIProxy { + proxyClient.SetDisableKeepAlives(true) + } + + // Parse agent cache configurations + if config.Cache != nil { + cacheLogger := c.logger.Named("cache") // Create the API proxier apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ @@ -671,83 +674,102 @@ func (c *AgentCommand) Run(args []string) int { proxyVaultToken := !config.Cache.ForceAutoAuthToken // Create the request handler - cacheHandler := cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken) + cacheHandler = cache.CachingHandler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken) + } - var listeners []net.Listener + var listeners []net.Listener - // If there are templates, add an in-process listener - if len(config.Templates) > 0 { - config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType}) - } - for i, lnConfig := range config.Listeners { - var ln net.Listener - var tlsConf *tls.Config + // If there are templates, add an in-process listener + if len(config.Templates) > 0 { + config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType}) + } + for i, lnConfig := range config.Listeners { + var ln net.Listener + var tlsConf *tls.Config - if lnConfig.Type == listenerutil.BufConnType { - inProcListener := bufconn.Listen(1024 * 1024) + if lnConfig.Type == listenerutil.BufConnType { + inProcListener := bufconn.Listen(1024 * 1024) + if config.Cache != nil { config.Cache.InProcDialer = listenerutil.NewBufConnWrapper(inProcListener) - ln = inProcListener - } else { - ln, tlsConf, err = cache.StartListener(lnConfig) - if err != nil { - c.UI.Error(fmt.Sprintf("Error starting listener: %v", err)) - return 1 - } } + ln = inProcListener + } else { + ln, tlsConf, err = cache.StartListener(lnConfig) + if err != nil { + c.UI.Error(fmt.Sprintf("Error starting listener: %v", err)) + return 1 + } + } - listeners = append(listeners, ln) + listeners = append(listeners, ln) - // Parse 'require_request_header' listener config option, and wrap - // the request handler if necessary - muxHandler := cacheHandler - if lnConfig.RequireRequestHeader && ("metrics_only" != lnConfig.Role) { - muxHandler = verifyRequestHeader(muxHandler) - } + listenerLogger := c.logger.Named("listener") - // Create a muxer and add paths relevant for the lease cache layer - mux := http.NewServeMux() - quitEnabled := lnConfig.AgentAPI != nil && lnConfig.AgentAPI.EnableQuit + var muxHandler http.Handler - mux.Handle(consts.AgentPathMetrics, c.handleMetrics()) - if "metrics_only" != lnConfig.Role { - mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle(consts.AgentPathQuit, c.handleQuit(quitEnabled)) - mux.Handle("/", muxHandler) + if cacheHandler != nil { + muxHandler = cacheHandler + } else { + apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ + Client: proxyClient, + Logger: listenerLogger.Named("apiproxy"), + }) + if err != nil { + c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err)) + return 1 } + muxHandler = cache.CachelessHandler(ctx, listenerLogger, apiProxy) + } - scheme := "https://" - if tlsConf == nil { - scheme = "http://" - } - if ln.Addr().Network() == "unix" { - scheme = "unix://" - } + // Parse 'require_request_header' listener config option, and wrap + // the request handler if necessary + if lnConfig.RequireRequestHeader && ("metrics_only" != lnConfig.Role) { + muxHandler = verifyRequestHeader(muxHandler) + } - infoKey := fmt.Sprintf("api address %d", i+1) - info[infoKey] = scheme + ln.Addr().String() - infoKeys = append(infoKeys, infoKey) - - server := &http.Server{ - Addr: ln.Addr().String(), - TLSConfig: tlsConf, - Handler: mux, - ReadHeaderTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - IdleTimeout: 5 * time.Minute, - ErrorLog: cacheLogger.StandardLogger(nil), - } + // Create a muxer and add paths relevant for the lease cache layer + mux := http.NewServeMux() + quitEnabled := lnConfig.AgentAPI != nil && lnConfig.AgentAPI.EnableQuit - go server.Serve(ln) + mux.Handle(consts.AgentPathMetrics, c.handleMetrics()) + if "metrics_only" != lnConfig.Role { + mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) + mux.Handle(consts.AgentPathQuit, c.handleQuit(quitEnabled)) + mux.Handle("/", muxHandler) } - // Ensure that listeners are closed at all the exits - listenerCloseFunc := func() { - for _, ln := range listeners { - ln.Close() - } + scheme := "https://" + if tlsConf == nil { + scheme = "http://" + } + if ln.Addr().Network() == "unix" { + scheme = "unix://" + } + + infoKey := fmt.Sprintf("api address %d", i+1) + info[infoKey] = scheme + ln.Addr().String() + infoKeys = append(infoKeys, infoKey) + + server := &http.Server{ + Addr: ln.Addr().String(), + TLSConfig: tlsConf, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 5 * time.Minute, + ErrorLog: listenerLogger.StandardLogger(nil), + } + + go server.Serve(ln) + } + + // Ensure that listeners are closed at all the exits + listenerCloseFunc := func() { + for _, ln := range listeners { + ln.Close() } - defer c.cleanupGuard.Do(listenerCloseFunc) } + defer c.cleanupGuard.Do(listenerCloseFunc) // Inform any tests that the server is ready if c.startedCh != nil { diff --git a/command/agent/cache/api_proxy_test.go b/command/agent/cache/api_proxy_test.go index b90f579c3f00..a50957490b77 100644 --- a/command/agent/cache/api_proxy_test.go +++ b/command/agent/cache/api_proxy_test.go @@ -47,6 +47,42 @@ func TestAPIProxy(t *testing.T) { } } +func TestAPIProxyNoCache(t *testing.T) { + cleanup, client, _, _ := setupClusterAndAgentNoCache(namespace.RootContext(nil), t, nil) + defer cleanup() + + proxier, err := NewAPIProxy(&APIProxyConfig{ + Client: client, + Logger: logging.NewVaultLogger(hclog.Trace), + }) + if err != nil { + t.Fatal(err) + } + + r := client.NewRequest("GET", "/v1/sys/health") + req, err := r.ToHTTP() + if err != nil { + t.Fatal(err) + } + + resp, err := proxier.Send(namespace.RootContext(nil), &SendRequest{ + Request: req, + }) + if err != nil { + t.Fatal(err) + } + + var result api.HealthResponse + err = jsonutil.DecodeJSONFromReader(resp.Response.Body, &result) + if err != nil { + t.Fatal(err) + } + + if !result.Initialized || result.Sealed || result.Standby { + t.Fatalf("bad sys/health response: %#v", result) + } +} + func TestAPIProxy_queryParams(t *testing.T) { // Set up an agent that points to a standby node for this particular test // since it needs to proxy a /sys/health?standbyok=true request to a standby diff --git a/command/agent/cache/cache_test.go b/command/agent/cache/cache_test.go index 8b2e1caa7155..be38513414ce 100644 --- a/command/agent/cache/cache_test.go +++ b/command/agent/cache/cache_test.go @@ -39,7 +39,15 @@ path "*" { // be deferred immediately along with two clients, one for direct cluster // communication and another to talk to the caching agent. func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { - return setupClusterAndAgentCommon(ctx, t, coreConfig, false) + return setupClusterAndAgentCommon(ctx, t, coreConfig, false, true) +} + +// setupClusterAndAgentNoCache is a helper func used to set up a test cluster and +// proxyi agent against the active node. It returns a cleanup func that should +// be deferred immediately along with two clients, one for direct cluster +// communication and another to talk to the caching agent. +func setupClusterAndAgentNoCache(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { + return setupClusterAndAgentCommon(ctx, t, coreConfig, false, false) } // setupClusterAndAgentOnStandby is a helper func used to set up a test cluster @@ -47,10 +55,10 @@ func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.C // should be deferred immediately along with two clients, one for direct cluster // communication and another to talk to the caching agent. func setupClusterAndAgentOnStandby(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { - return setupClusterAndAgentCommon(ctx, t, coreConfig, true) + return setupClusterAndAgentCommon(ctx, t, coreConfig, true, true) } -func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool) (func(), *api.Client, *api.Client, *LeaseCache) { +func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool, useCache bool) (func(), *api.Client, *api.Client, *LeaseCache) { t.Helper() if ctx == nil { @@ -121,45 +129,54 @@ func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *v origEnvVaultCACert := os.Getenv(api.EnvVaultCACert) os.Setenv(api.EnvVaultCACert, fmt.Sprintf("%s/ca_cert.pem", cluster.TempDir)) - cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache") - listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } + apiProxyLogger := logging.NewVaultLogger(hclog.Trace).Named("apiproxy") + // Create the API proxier apiProxy, err := NewAPIProxy(&APIProxyConfig{ Client: clienToUse, - Logger: cacheLogger.Named("apiproxy"), - }) - if err != nil { - t.Fatal(err) - } - - // Create the lease cache proxier and set its underlying proxier to - // the API proxier. - leaseCache, err := NewLeaseCache(&LeaseCacheConfig{ - Client: clienToUse, - BaseContext: ctx, - Proxier: apiProxy, - Logger: cacheLogger.Named("leasecache"), + Logger: apiProxyLogger, }) if err != nil { t.Fatal(err) } - // Create a muxer and add paths relevant for the lease cache layer + // Create a muxer and add paths relevant for the lease cache layer and API proxy layer mux := http.NewServeMux() - mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, nil, true)) + var leaseCache *LeaseCache + if useCache { + cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache") + + // Create the lease cache proxier and set its underlying proxier to + // the API proxier. + leaseCache, err = NewLeaseCache(&LeaseCacheConfig{ + Client: clienToUse, + BaseContext: ctx, + Proxier: apiProxy, + Logger: cacheLogger.Named("leasecache"), + }) + if err != nil { + t.Fatal(err) + } + + mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx)) + + mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, nil, true)) + } else { + mux.Handle("/", CachelessHandler(ctx, apiProxyLogger, apiProxy)) + } + server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute, - ErrorLog: cacheLogger.StandardLogger(nil), + ErrorLog: apiProxyLogger.StandardLogger(nil), } go server.Serve(listener) @@ -248,7 +265,7 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) { mux := http.NewServeMux() mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) + mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, @@ -337,7 +354,7 @@ func TestCache_AutoAuthClientTokenProxyStripping(t *testing.T) { mux := http.NewServeMux() // mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", Handler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) + mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, diff --git a/command/agent/cache/handler.go b/command/agent/cache/handler.go index 60f9e046a0fa..baeda2838a1a 100644 --- a/command/agent/cache/handler.go +++ b/command/agent/cache/handler.go @@ -20,7 +20,61 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { +func CachelessHandler(ctx context.Context, logger hclog.Logger, proxier Proxier) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + logger.Info("received request", "method", r.Method, "path", r.URL.Path) + + token := r.Header.Get(consts.AuthHeaderName) + + // Parse and reset body. + reqBody, err := io.ReadAll(r.Body) + if err != nil { + logger.Error("failed to read request body") + logical.RespondError(w, http.StatusInternalServerError, errors.New("failed to read request body")) + return + } + if r.Body != nil { + r.Body.Close() + } + r.Body = io.NopCloser(bytes.NewReader(reqBody)) + req := &SendRequest{ + Token: token, + Request: r, + RequestBody: reqBody, + } + + resp, err := proxier.Send(ctx, req) + if err != nil { + // If this is an api.Response error, don't wrap the response. + if resp != nil && resp.Response.Error() != nil { + copyHeader(w.Header(), resp.Response.Header) + w.WriteHeader(resp.Response.StatusCode) + io.Copy(w, resp.Response.Body) + metrics.IncrCounter([]string{"agent", "proxy", "client_error"}, 1) + } else { + metrics.IncrCounter([]string{"agent", "proxy", "error"}, 1) + logical.RespondError(w, http.StatusInternalServerError, fmt.Errorf("failed to get the response: %w", err)) + } + return + } + + if err != nil { + logical.RespondError(w, http.StatusInternalServerError, fmt.Errorf("failed to process token lookup response: %w", err)) + return + } + + defer resp.Response.Body.Close() + + // Set headers + setHeaders(w, resp) + + // Set response body + io.Copy(w, resp.Response.Body) + return + }) +} + +func CachingHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("received request", "method", r.Method, "path", r.URL.Path) @@ -36,7 +90,7 @@ func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSin } // Parse and reset body. - reqBody, err := ioutil.ReadAll(r.Body) + reqBody, err := io.ReadAll(r.Body) if err != nil { logger.Error("failed to read request body") logical.RespondError(w, http.StatusInternalServerError, errors.New("failed to read request body")) @@ -45,7 +99,7 @@ func Handler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSin if r.Body != nil { r.Body.Close() } - r.Body = ioutil.NopCloser(bytes.NewReader(reqBody)) + r.Body = io.NopCloser(bytes.NewReader(reqBody)) req := &SendRequest{ Token: token, Request: r, diff --git a/command/agent/cache/lease_cache.go b/command/agent/cache/lease_cache.go index 9bc79d4a2713..87bfacd97ec1 100644 --- a/command/agent/cache/lease_cache.go +++ b/command/agent/cache/lease_cache.go @@ -568,6 +568,11 @@ func computeIndexID(req *SendRequest) (string, error) { // HandleCacheClear returns a handlerFunc that can perform cache clearing operations. func (c *LeaseCache) HandleCacheClear(ctx context.Context) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // If the cache is not enabled, return a 200 + if c == nil { + return + } + // Only handle POST/PUT requests switch r.Method { case http.MethodPost: diff --git a/command/agent/cache_end_to_end_test.go b/command/agent/cache_end_to_end_test.go index 4ad056a850cc..3f50159796db 100644 --- a/command/agent/cache_end_to_end_test.go +++ b/command/agent/cache_end_to_end_test.go @@ -315,7 +315,7 @@ func TestCache_UsingAutoAuthToken(t *testing.T) { mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) // Passing a non-nil inmemsink tells the agent to use the auto-auth token - mux.Handle("/", cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, true)) + mux.Handle("/", cache.CachingHandler(ctx, cacheLogger, leaseCache, inmemSink, true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 3d8b96d3e0ff..ba9722e835ad 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -30,11 +30,11 @@ type Config struct { TemplateConfig *TemplateConfig `hcl:"template_config"` Templates []*ctconfig.TemplateConfig `hcl:"templates"` DisableIdleConns []string `hcl:"disable_idle_connections"` - DisableIdleConnsCaching bool `hcl:"-"` + DisableIdleConnsAPIProxy bool `hcl:"-"` DisableIdleConnsTemplating bool `hcl:"-"` DisableIdleConnsAutoAuth bool `hcl:"-"` DisableKeepAlives []string `hcl:"disable_keep_alives"` - DisableKeepAlivesCaching bool `hcl:"-"` + DisableKeepAlivesAPIProxy bool `hcl:"-"` DisableKeepAlivesTemplating bool `hcl:"-"` DisableKeepAlivesAutoAuth bool `hcl:"-"` LogFile string `hcl:"log_file"` @@ -286,7 +286,9 @@ func LoadConfig(path string) (*Config, error) { case "auto-auth": result.DisableIdleConnsAutoAuth = true case "caching": - result.DisableIdleConnsCaching = true + result.DisableIdleConnsAPIProxy = true + case "proxying": + result.DisableIdleConnsAPIProxy = true case "templating": result.DisableIdleConnsTemplating = true case "": @@ -308,7 +310,9 @@ func LoadConfig(path string) (*Config, error) { case "auto-auth": result.DisableKeepAlivesAutoAuth = true case "caching": - result.DisableKeepAlivesCaching = true + result.DisableKeepAlivesAPIProxy = true + case "proxying": + result.DisableKeepAlivesAPIProxy = true case "templating": result.DisableKeepAlivesTemplating = true case "": diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index a005a420b7a4..1511eafe20fc 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -1102,8 +1102,8 @@ func TestLoadConfigFile_Disable_Idle_Conns_All(t *testing.T) { SharedConfig: &configutil.SharedConfig{ PidFile: "./pidfile", }, - DisableIdleConns: []string{"auto-auth", "caching", "templating"}, - DisableIdleConnsCaching: true, + DisableIdleConns: []string{"auto-auth", "caching", "templating", "proxying"}, + DisableIdleConnsAPIProxy: true, DisableIdleConnsAutoAuth: true, DisableIdleConnsTemplating: true, AutoAuth: &AutoAuth{ @@ -1152,7 +1152,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Auto_Auth(t *testing.T) { PidFile: "./pidfile", }, DisableIdleConns: []string{"auto-auth"}, - DisableIdleConnsCaching: false, + DisableIdleConnsAPIProxy: false, DisableIdleConnsAutoAuth: true, DisableIdleConnsTemplating: false, AutoAuth: &AutoAuth{ @@ -1201,7 +1201,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Templating(t *testing.T) { PidFile: "./pidfile", }, DisableIdleConns: []string{"templating"}, - DisableIdleConnsCaching: false, + DisableIdleConnsAPIProxy: false, DisableIdleConnsAutoAuth: false, DisableIdleConnsTemplating: true, AutoAuth: &AutoAuth{ @@ -1250,7 +1250,56 @@ func TestLoadConfigFile_Disable_Idle_Conns_Caching(t *testing.T) { PidFile: "./pidfile", }, DisableIdleConns: []string{"caching"}, - DisableIdleConnsCaching: true, + DisableIdleConnsAPIProxy: true, + DisableIdleConnsAutoAuth: false, + DisableIdleConnsTemplating: false, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Idle_Conns_Proxying(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-idle-connections-proxying.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableIdleConns: []string{"proxying"}, + DisableIdleConnsAPIProxy: true, DisableIdleConnsAutoAuth: false, DisableIdleConnsTemplating: false, AutoAuth: &AutoAuth{ @@ -1299,7 +1348,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Empty(t *testing.T) { PidFile: "./pidfile", }, DisableIdleConns: []string{}, - DisableIdleConnsCaching: false, + DisableIdleConnsAPIProxy: false, DisableIdleConnsAutoAuth: false, DisableIdleConnsTemplating: false, AutoAuth: &AutoAuth{ @@ -1354,7 +1403,7 @@ func TestLoadConfigFile_Disable_Idle_Conns_Env(t *testing.T) { PidFile: "./pidfile", }, DisableIdleConns: []string{"auto-auth", "caching", "templating"}, - DisableIdleConnsCaching: true, + DisableIdleConnsAPIProxy: true, DisableIdleConnsAutoAuth: true, DisableIdleConnsTemplating: true, AutoAuth: &AutoAuth{ @@ -1409,8 +1458,8 @@ func TestLoadConfigFile_Disable_Keep_Alives_All(t *testing.T) { SharedConfig: &configutil.SharedConfig{ PidFile: "./pidfile", }, - DisableKeepAlives: []string{"auto-auth", "caching", "templating"}, - DisableKeepAlivesCaching: true, + DisableKeepAlives: []string{"auto-auth", "caching", "templating", "proxying"}, + DisableKeepAlivesAPIProxy: true, DisableKeepAlivesAutoAuth: true, DisableKeepAlivesTemplating: true, AutoAuth: &AutoAuth{ @@ -1459,7 +1508,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Auto_Auth(t *testing.T) { PidFile: "./pidfile", }, DisableKeepAlives: []string{"auto-auth"}, - DisableKeepAlivesCaching: false, + DisableKeepAlivesAPIProxy: false, DisableKeepAlivesAutoAuth: true, DisableKeepAlivesTemplating: false, AutoAuth: &AutoAuth{ @@ -1508,7 +1557,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Templating(t *testing.T) { PidFile: "./pidfile", }, DisableKeepAlives: []string{"templating"}, - DisableKeepAlivesCaching: false, + DisableKeepAlivesAPIProxy: false, DisableKeepAlivesAutoAuth: false, DisableKeepAlivesTemplating: true, AutoAuth: &AutoAuth{ @@ -1557,7 +1606,56 @@ func TestLoadConfigFile_Disable_Keep_Alives_Caching(t *testing.T) { PidFile: "./pidfile", }, DisableKeepAlives: []string{"caching"}, - DisableKeepAlivesCaching: true, + DisableKeepAlivesAPIProxy: true, + DisableKeepAlivesAutoAuth: false, + DisableKeepAlivesTemplating: false, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Proxying(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-proxying.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"proxying"}, + DisableKeepAlivesAPIProxy: true, DisableKeepAlivesAutoAuth: false, DisableKeepAlivesTemplating: false, AutoAuth: &AutoAuth{ @@ -1606,7 +1704,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Empty(t *testing.T) { PidFile: "./pidfile", }, DisableKeepAlives: []string{}, - DisableKeepAlivesCaching: false, + DisableKeepAlivesAPIProxy: false, DisableKeepAlivesAutoAuth: false, DisableKeepAlivesTemplating: false, AutoAuth: &AutoAuth{ @@ -1661,7 +1759,7 @@ func TestLoadConfigFile_Disable_Keep_Alives_Env(t *testing.T) { PidFile: "./pidfile", }, DisableKeepAlives: []string{"auto-auth", "caching", "templating"}, - DisableKeepAlivesCaching: true, + DisableKeepAlivesAPIProxy: true, DisableKeepAlivesAutoAuth: true, DisableKeepAlivesTemplating: true, AutoAuth: &AutoAuth{ diff --git a/command/agent/config/test-fixtures/config-disable-idle-connections-all.hcl b/command/agent/config/test-fixtures/config-disable-idle-connections-all.hcl index 69ff548f5561..94e8cc827f3c 100644 --- a/command/agent/config/test-fixtures/config-disable-idle-connections-all.hcl +++ b/command/agent/config/test-fixtures/config-disable-idle-connections-all.hcl @@ -1,5 +1,5 @@ pid_file = "./pidfile" -disable_idle_connections = ["auto-auth","caching","templating"] +disable_idle_connections = ["auto-auth","caching","templating","proxying"] auto_auth { method { diff --git a/command/agent/config/test-fixtures/config-disable-idle-connections-proxying.hcl b/command/agent/config/test-fixtures/config-disable-idle-connections-proxying.hcl new file mode 100644 index 000000000000..8c2c6db67400 --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-idle-connections-proxying.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_idle_connections = ["proxying"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl index 9b22cfd8be67..6e498f756d39 100644 --- a/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl @@ -1,5 +1,5 @@ pid_file = "./pidfile" -disable_keep_alives = ["auto-auth","caching","templating"] +disable_keep_alives = ["auto-auth","caching","templating","proxying"] auto_auth { method { diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-proxying.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-proxying.hcl new file mode 100644 index 000000000000..8363cb58f132 --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-proxying.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["proxying"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/website/content/docs/agent/caching/index.mdx b/website/content/docs/agent/caching/index.mdx index e46fa212f9f4..6cc96f17bca8 100644 --- a/website/content/docs/agent/caching/index.mdx +++ b/website/content/docs/agent/caching/index.mdx @@ -134,7 +134,8 @@ belonging to the cached response, the `request_path` that resulted in the cached response, the `lease` that is attached to the cached response, the `namespace` to which the cached response belongs to, and a few more. This API exposes some factors through which associated cache entries are fetched and -evicted. +evicted. For listeners without caching enabled, this API will still be available, +but will do nothing (there is no cache to clear) and will return a `200` response. | Method | Path | Produces | | :----- | :---------------------- | :--------------------- | @@ -251,7 +252,7 @@ or the default role, `default`, which serves everything (including metrics). ### Example Configuration -Here is an example of a cache configuration alongside a listener that only serves metrics. +Here is an example of a cache configuration alongside a regular listener, and a listener that only serves metrics. ```hcl # Other Vault Agent configuration blocks @@ -261,6 +262,11 @@ cache { use_auto_auth_token = true } +listener "tcp" { + address = "127.0.0.1:8100" + tls_disable = true +} + listener "tcp" { address = "127.0.0.1:3000" tls_disable = true diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index e087878fa8d8..2226c45a8043 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -151,11 +151,13 @@ These are the currently-available general configuration option: token was retrieved and all sinks successfully wrote it - `disable_idle_connections` `(string array: [])` - A list of strings that disables idle connections for various features in Vault Agent. - Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS` + Valid values include: `auto-auth`, `caching`, `proxying`, and `templating`. `proxying` configures this for the API proxy, which is + identical in function to `caching` for historical reasons. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS` environment variable as a comma separated string. This environment variable will override any values found in a configuration file. - `disable_keep_alives` `(string array: [])` - A list of strings that disables keep alives for various features in Vault Agent. - Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES` + Valid values include: `auto-auth`, `caching`, `proxying`, and `templating`. `proxying` configures this for the API proxy, which is + dentical in function to `caching` for historical reasons. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES` environment variable as a comma separated string. This environment variable will override any values found in a configuration file. - `template` ([template][template]: ) - Specifies options used for templating Vault secrets to files. @@ -171,9 +173,9 @@ These are the currently-available general configuration option: There can at most be one top level `vault` block and it has the following configuration entries: -- `address` `(string: )` - The address of the Vault server to - connect to. This should be a Fully Qualified Domain Name (FQDN) or IP - such as `https://vault-fqdn:8200` or `https://172.16.9.8:8200`. +- `address` `(string: )` - The address of the Vault server to + connect to. This should be a Fully Qualified Domain Name (FQDN) or IP + such as `https://vault-fqdn:8200` or `https://172.16.9.8:8200`. This value can be overridden by setting the `VAULT_ADDR` environment variable. - `ca_cert` `(string: )` - Path on the local disk to a single PEM-encoded @@ -243,9 +245,10 @@ to address in the future. ### listener Stanza -Vault Agent supports one or more [listener][listener_main] stanzas. In addition -to the standard listener configuration, an Agent's listener configuration also -supports the following: +Vault Agent supports one or more [listener][listener_main] stanzas. Listeners +can be configured with or without [caching][caching], but will use the cache if it +has been configured. In addition to the standard listener configuration, +an Agent's listener configuration also supports the following: - `require_request_header` `(bool: false)` - Require that all incoming HTTP requests on this listener must have an `X-Vault-Request: true` header entry. @@ -253,6 +256,11 @@ supports the following: Request Forgery attacks. Requests on the listener that do not have the proper `X-Vault-Request` header will fail, with a HTTP response status code of `412: Precondition Failed`. +- `role` `(string: default)` - `role` determines which APIs the listener serves. + It can be configured to `metrics_only` to serve only metrics, or the default role, `default`, + which serves everything (including metrics). The `require_request_header` does not apply + to `metrics_only` listeners. + - `agent_api` ([agent_api][agent-api]: ) - Manages optional Agent API endpoints. #### agent_api Stanza @@ -379,4 +387,4 @@ template { [listener]: /docs/agent#listener-stanza [listener_main]: /docs/configuration/listener/tcp [winsvc]: /docs/agent/winsvc -[telemetry]: /docs/configuration/telemetry \ No newline at end of file +[telemetry]: /docs/configuration/telemetry From 2bad6bbb60be9161c465718aa1e8fd1c2532254d Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Mon, 28 Nov 2022 16:28:45 -0500 Subject: [PATCH 02/10] VAULT-11510 fix order of imports --- command/agent.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/command/agent.go b/command/agent.go index 5a83db3467d6..040bf3d21aee 100644 --- a/command/agent.go +++ b/command/agent.go @@ -16,9 +16,6 @@ import ( "sync" "time" - "github.com/hashicorp/vault/internalshared/listenerutil" - "google.golang.org/grpc/test/bufconn" - systemd "github.com/coreos/go-systemd/daemon" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/gatedwriter" @@ -47,6 +44,7 @@ import ( "github.com/hashicorp/vault/helper/logging" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/internalshared/configutil" + "github.com/hashicorp/vault/internalshared/listenerutil" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/useragent" "github.com/hashicorp/vault/sdk/logical" @@ -55,6 +53,7 @@ import ( "github.com/mitchellh/cli" "github.com/oklog/run" "github.com/posener/complete" + "google.golang.org/grpc/test/bufconn" ) var ( From 68c3c7acab067f2b8115f225a464dbcf6f6eba8c Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Tue, 29 Nov 2022 09:15:40 -0500 Subject: [PATCH 03/10] VAULT-11510 changelog --- changelog/18137.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/18137.txt diff --git a/changelog/18137.txt b/changelog/18137.txt new file mode 100644 index 000000000000..f262f96f3521 --- /dev/null +++ b/changelog/18137.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: Configured Vault Agent listeners now listen without the need for caching to be configured. +``` From 242f644b3fea76613bd2e6db72033eeabf23df18 Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Wed, 30 Nov 2022 09:54:31 -0500 Subject: [PATCH 04/10] VAULT-11510 typo and better switch --- command/agent/cache/cache_test.go | 2 +- command/agent/config/config.go | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/command/agent/cache/cache_test.go b/command/agent/cache/cache_test.go index be38513414ce..ee3cb47591d9 100644 --- a/command/agent/cache/cache_test.go +++ b/command/agent/cache/cache_test.go @@ -43,7 +43,7 @@ func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.C } // setupClusterAndAgentNoCache is a helper func used to set up a test cluster and -// proxyi agent against the active node. It returns a cleanup func that should +// proxying agent against the active node. It returns a cleanup func that should // be deferred immediately along with two clients, one for direct cluster // communication and another to talk to the caching agent. func setupClusterAndAgentNoCache(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 579827864a22..745a804ebd84 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -284,9 +284,7 @@ func LoadConfig(path string) (*Config, error) { switch subsystem { case "auto-auth": result.DisableIdleConnsAutoAuth = true - case "caching": - result.DisableIdleConnsAPIProxy = true - case "proxying": + case "caching", "proxying": result.DisableIdleConnsAPIProxy = true case "templating": result.DisableIdleConnsTemplating = true @@ -308,9 +306,7 @@ func LoadConfig(path string) (*Config, error) { switch subsystem { case "auto-auth": result.DisableKeepAlivesAutoAuth = true - case "caching": - result.DisableKeepAlivesAPIProxy = true - case "proxying": + case "caching", "proxying": result.DisableKeepAlivesAPIProxy = true case "templating": result.DisableKeepAlivesTemplating = true From dfd424b69cf3d3190106dd1c5693e2477b178300 Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Wed, 30 Nov 2022 11:48:27 -0500 Subject: [PATCH 05/10] VAULT-11510 update name --- command/agent.go | 4 ++-- command/agent/cache/cache_test.go | 8 ++++---- command/agent/cache/handler.go | 4 ++-- command/agent/cache_end_to_end_test.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/command/agent.go b/command/agent.go index f7eb3aad5636..fe5942282b8e 100644 --- a/command/agent.go +++ b/command/agent.go @@ -676,7 +676,7 @@ func (c *AgentCommand) Run(args []string) int { proxyVaultToken := !config.Cache.ForceAutoAuthToken // Create the request handler - cacheHandler = cache.CachingHandler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken) + cacheHandler = cache.CachingProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken) } var listeners []net.Listener @@ -720,7 +720,7 @@ func (c *AgentCommand) Run(args []string) int { c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err)) return 1 } - muxHandler = cache.CachelessHandler(ctx, listenerLogger, apiProxy) + muxHandler = cache.ProxyHandler(ctx, listenerLogger, apiProxy) } // Parse 'require_request_header' listener config option, and wrap diff --git a/command/agent/cache/cache_test.go b/command/agent/cache/cache_test.go index ee3cb47591d9..f72d1f3b417f 100644 --- a/command/agent/cache/cache_test.go +++ b/command/agent/cache/cache_test.go @@ -166,9 +166,9 @@ func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *v mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, nil, true)) + mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, nil, true)) } else { - mux.Handle("/", CachelessHandler(ctx, apiProxyLogger, apiProxy)) + mux.Handle("/", ProxyHandler(ctx, apiProxyLogger, apiProxy)) } server := &http.Server{ @@ -265,7 +265,7 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) { mux := http.NewServeMux() mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) + mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, @@ -354,7 +354,7 @@ func TestCache_AutoAuthClientTokenProxyStripping(t *testing.T) { mux := http.NewServeMux() // mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", CachingHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) + mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, diff --git a/command/agent/cache/handler.go b/command/agent/cache/handler.go index baeda2838a1a..a93f2f83cf0f 100644 --- a/command/agent/cache/handler.go +++ b/command/agent/cache/handler.go @@ -20,7 +20,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func CachelessHandler(ctx context.Context, logger hclog.Logger, proxier Proxier) http.Handler { +func ProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("received request", "method", r.Method, "path", r.URL.Path) @@ -74,7 +74,7 @@ func CachelessHandler(ctx context.Context, logger hclog.Logger, proxier Proxier) }) } -func CachingHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { +func CachingProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("received request", "method", r.Method, "path", r.URL.Path) diff --git a/command/agent/cache_end_to_end_test.go b/command/agent/cache_end_to_end_test.go index 3f50159796db..8a3400cea8c1 100644 --- a/command/agent/cache_end_to_end_test.go +++ b/command/agent/cache_end_to_end_test.go @@ -315,7 +315,7 @@ func TestCache_UsingAutoAuthToken(t *testing.T) { mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) // Passing a non-nil inmemsink tells the agent to use the auto-auth token - mux.Handle("/", cache.CachingHandler(ctx, cacheLogger, leaseCache, inmemSink, true)) + mux.Handle("/", cache.CachingProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, From 3292202ec31c67b5374862e05e47968ada60a1d8 Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Wed, 30 Nov 2022 16:37:22 -0500 Subject: [PATCH 06/10] VAULT-11510 New api_proxy stanza to configure API proxy --- command/agent.go | 130 +++++++----- command/agent/cache/api_proxy_test.go | 195 ++++++++++++++++++ command/agent/cache/cache_test.go | 191 +---------------- command/agent/cache/handler.go | 56 +---- command/agent/cache/proxy.go | 6 +- command/agent/cache_end_to_end_test.go | 2 +- command/agent/config/config.go | 98 ++++++++- command/agent/config/config_test.go | 112 +++++++++- .../bad-config-api_proxy-cache.hcl | 23 +++ ...i_proxy-auto_auth-all-api_proxy-config.hcl | 21 ++ .../config-cache-auto_auth-true.hcl | 2 +- .../config-consistency-apiproxy.hcl | 9 + 12 files changed, 537 insertions(+), 308 deletions(-) create mode 100644 command/agent/config/test-fixtures/bad-config-api_proxy-cache.hcl create mode 100644 command/agent/config/test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl create mode 100644 command/agent/config/test-fixtures/config-consistency-apiproxy.hcl diff --git a/command/agent.go b/command/agent.go index fe5942282b8e..e50c9610733f 100644 --- a/command/agent.go +++ b/command/agent.go @@ -16,6 +16,8 @@ import ( "sync" "time" + "github.com/hashicorp/vault/command/agent/sink/inmem" + systemd "github.com/coreos/go-systemd/daemon" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/gatedwriter" @@ -39,7 +41,6 @@ import ( agentConfig "github.com/hashicorp/vault/command/agent/config" "github.com/hashicorp/vault/command/agent/sink" "github.com/hashicorp/vault/command/agent/sink/file" - "github.com/hashicorp/vault/command/agent/sink/inmem" "github.com/hashicorp/vault/command/agent/template" "github.com/hashicorp/vault/command/agent/winsvc" "github.com/hashicorp/vault/helper/logging" @@ -421,10 +422,37 @@ func (c *AgentCommand) Run(args []string) int { enforceConsistency := cache.EnforceConsistencyNever whenInconsistent := cache.WhenInconsistentFail + if config.APIProxy != nil { + switch config.APIProxy.EnforceConsistency { + case "always": + enforceConsistency = cache.EnforceConsistencyAlways + case "never", "": + default: + c.UI.Error(fmt.Sprintf("Unknown api_proxy setting for enforce_consistency: %q", config.APIProxy.EnforceConsistency)) + return 1 + } + + switch config.APIProxy.WhenInconsistent { + case "retry": + whenInconsistent = cache.WhenInconsistentRetry + case "forward": + whenInconsistent = cache.WhenInconsistentForward + case "fail", "": + default: + c.UI.Error(fmt.Sprintf("Unknown api_proxy setting for when_inconsistent: %q", config.APIProxy.WhenInconsistent)) + return 1 + } + } + // Keep Cache configuration for legacy reasons, but error if defined alongside API Proxy if config.Cache != nil { switch config.Cache.EnforceConsistency { case "always": - enforceConsistency = cache.EnforceConsistencyAlways + if enforceConsistency != cache.EnforceConsistencyNever { + c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") + return 1 + } else { + enforceConsistency = cache.EnforceConsistencyAlways + } case "never", "": default: c.UI.Error(fmt.Sprintf("Unknown cache setting for enforce_consistency: %q", config.Cache.EnforceConsistency)) @@ -433,9 +461,19 @@ func (c *AgentCommand) Run(args []string) int { switch config.Cache.WhenInconsistent { case "retry": - whenInconsistent = cache.WhenInconsistentRetry + if whenInconsistent != cache.WhenInconsistentFail { + c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") + return 1 + } else { + whenInconsistent = cache.WhenInconsistentRetry + } case "forward": - whenInconsistent = cache.WhenInconsistentForward + if whenInconsistent != cache.WhenInconsistentFail { + c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") + return 1 + } else { + whenInconsistent = cache.WhenInconsistentForward + } case "fail", "": default: c.UI.Error(fmt.Sprintf("Unknown cache setting for when_inconsistent: %q", config.Cache.WhenInconsistent)) @@ -466,7 +504,6 @@ func (c *AgentCommand) Run(args []string) int { var leaseCache *cache.LeaseCache var previousToken string - var cacheHandler http.Handler proxyClient, err := client.CloneWithHeaders() if err != nil { @@ -482,22 +519,24 @@ func (c *AgentCommand) Run(args []string) int { proxyClient.SetDisableKeepAlives(true) } + apiProxyLogger := c.logger.Named("apiproxy") + + // The API proxy to be used, if listeners are configured + apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ + Client: proxyClient, + Logger: apiProxyLogger, + EnforceConsistency: enforceConsistency, + WhenInconsistentAction: whenInconsistent, + }) + if err != nil { + c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err)) + return 1 + } + // Parse agent cache configurations if config.Cache != nil { cacheLogger := c.logger.Named("cache") - // Create the API proxier - apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ - Client: proxyClient, - Logger: cacheLogger.Named("apiproxy"), - EnforceConsistency: enforceConsistency, - WhenInconsistentAction: whenInconsistent, - }) - if err != nil { - c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err)) - return 1 - } - // Create the lease cache proxier and set its underlying proxier to // the API proxier. leaseCache, err = cache.NewLeaseCache(&cache.LeaseCacheConfig{ @@ -656,27 +695,6 @@ func (c *AgentCommand) Run(args []string) int { leaseCache.SetPersistentStorage(ps) } } - - var inmemSink sink.Sink - if config.Cache.UseAutoAuthToken { - cacheLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink") - inmemSink, err = inmem.New(&sink.SinkConfig{ - Logger: cacheLogger, - }, leaseCache) - if err != nil { - c.UI.Error(fmt.Sprintf("Error creating inmem sink for cache: %v", err)) - return 1 - } - sinks = append(sinks, &sink.SinkConfig{ - Logger: cacheLogger, - Sink: inmemSink, - }) - } - - proxyVaultToken := !config.Cache.ForceAutoAuthToken - - // Create the request handler - cacheHandler = cache.CachingProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken) } var listeners []net.Listener @@ -705,24 +723,28 @@ func (c *AgentCommand) Run(args []string) int { listeners = append(listeners, ln) - listenerLogger := c.logger.Named("listener") - - var muxHandler http.Handler - - if cacheHandler != nil { - muxHandler = cacheHandler - } else { - apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ - Client: proxyClient, - Logger: listenerLogger.Named("apiproxy"), - }) - if err != nil { - c.UI.Error(fmt.Sprintf("Error creating API proxy: %v", err)) - return 1 + proxyVaultToken := true + var inmemSink sink.Sink + if config.APIProxy != nil { + if config.APIProxy.UseAutoAuthToken { + apiProxyLogger.Debug("auto-auth token is allowed to be used; configuring inmem sink") + inmemSink, err = inmem.New(&sink.SinkConfig{ + Logger: apiProxyLogger, + }, leaseCache) + if err != nil { + c.UI.Error(fmt.Sprintf("Error creating inmem sink for cache: %v", err)) + return 1 + } + sinks = append(sinks, &sink.SinkConfig{ + Logger: apiProxyLogger, + Sink: inmemSink, + }) } - muxHandler = cache.ProxyHandler(ctx, listenerLogger, apiProxy) + proxyVaultToken = !config.APIProxy.ForceAutoAuthToken } + muxHandler := cache.ProxyHandler(ctx, apiProxyLogger, apiProxy, inmemSink, proxyVaultToken) + // Parse 'require_request_header' listener config option, and wrap // the request handler if necessary if lnConfig.RequireRequestHeader && ("metrics_only" != lnConfig.Role) { @@ -759,7 +781,7 @@ func (c *AgentCommand) Run(args []string) int { ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute, - ErrorLog: listenerLogger.StandardLogger(nil), + ErrorLog: apiProxyLogger.StandardLogger(nil), } go server.Serve(ln) diff --git a/command/agent/cache/api_proxy_test.go b/command/agent/cache/api_proxy_test.go index a50957490b77..aec0b72d1fc9 100644 --- a/command/agent/cache/api_proxy_test.go +++ b/command/agent/cache/api_proxy_test.go @@ -1,8 +1,18 @@ package cache import ( + "context" + "fmt" + "net" "net/http" + "os" "testing" + "time" + + "github.com/hashicorp/vault/builtin/credential/userpass" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault" "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/api" @@ -11,6 +21,12 @@ import ( "github.com/hashicorp/vault/sdk/helper/logging" ) +const policyAdmin = ` +path "*" { + capabilities = ["sudo", "create", "read", "update", "delete", "list"] +} +` + func TestAPIProxy(t *testing.T) { cleanup, client, _, _ := setupClusterAndAgent(namespace.RootContext(nil), t, nil) defer cleanup() @@ -129,3 +145,182 @@ func TestAPIProxy_queryParams(t *testing.T) { t.Fatalf("exptected standby to return 200, got: %v", resp.Response.StatusCode) } } + +// setupClusterAndAgent is a helper func used to set up a test cluster and +// caching agent against the active node. It returns a cleanup func that should +// be deferred immediately along with two clients, one for direct cluster +// communication and another to talk to the caching agent. +func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { + return setupClusterAndAgentCommon(ctx, t, coreConfig, false, true) +} + +// setupClusterAndAgentNoCache is a helper func used to set up a test cluster and +// proxying agent against the active node. It returns a cleanup func that should +// be deferred immediately along with two clients, one for direct cluster +// communication and another to talk to the caching agent. +func setupClusterAndAgentNoCache(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { + return setupClusterAndAgentCommon(ctx, t, coreConfig, false, false) +} + +// setupClusterAndAgentOnStandby is a helper func used to set up a test cluster +// and caching agent against a standby node. It returns a cleanup func that +// should be deferred immediately along with two clients, one for direct cluster +// communication and another to talk to the caching agent. +func setupClusterAndAgentOnStandby(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { + return setupClusterAndAgentCommon(ctx, t, coreConfig, true, true) +} + +func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool, useCache bool) (func(), *api.Client, *api.Client, *LeaseCache) { + t.Helper() + + if ctx == nil { + ctx = context.Background() + } + + // Handle sane defaults + if coreConfig == nil { + coreConfig = &vault.CoreConfig{ + DisableMlock: true, + DisableCache: true, + Logger: logging.NewVaultLogger(hclog.Trace), + } + } + + // Always set up the userpass backend since we use that to generate an admin + // token for the client that will make proxied requests to through the agent. + if coreConfig.CredentialBackends == nil || coreConfig.CredentialBackends["userpass"] == nil { + coreConfig.CredentialBackends = map[string]logical.Factory{ + "userpass": userpass.Factory, + } + } + + // Init new test cluster + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + + cores := cluster.Cores + vault.TestWaitActive(t, cores[0].Core) + + activeClient := cores[0].Client + standbyClient := cores[1].Client + + // clienToUse is the client for the agent to point to. + clienToUse := activeClient + if onStandby { + clienToUse = standbyClient + } + + // Add an admin policy + if err := activeClient.Sys().PutPolicy("admin", policyAdmin); err != nil { + t.Fatal(err) + } + + // Set up the userpass auth backend and an admin user. Used for getting a token + // for the agent later down in this func. + err := activeClient.Sys().EnableAuthWithOptions("userpass", &api.EnableAuthOptions{ + Type: "userpass", + }) + if err != nil { + t.Fatal(err) + } + + _, err = activeClient.Logical().Write("auth/userpass/users/foo", map[string]interface{}{ + "password": "bar", + "policies": []string{"admin"}, + }) + if err != nil { + t.Fatal(err) + } + + // Set up env vars for agent consumption + origEnvVaultAddress := os.Getenv(api.EnvVaultAddress) + os.Setenv(api.EnvVaultAddress, clienToUse.Address()) + + origEnvVaultCACert := os.Getenv(api.EnvVaultCACert) + os.Setenv(api.EnvVaultCACert, fmt.Sprintf("%s/ca_cert.pem", cluster.TempDir)) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + apiProxyLogger := logging.NewVaultLogger(hclog.Trace).Named("apiproxy") + + // Create the API proxier + apiProxy, err := NewAPIProxy(&APIProxyConfig{ + Client: clienToUse, + Logger: apiProxyLogger, + }) + if err != nil { + t.Fatal(err) + } + + // Create a muxer and add paths relevant for the lease cache layer and API proxy layer + mux := http.NewServeMux() + + var leaseCache *LeaseCache + if useCache { + cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache") + + // Create the lease cache proxier and set its underlying proxier to + // the API proxier. + leaseCache, err = NewLeaseCache(&LeaseCacheConfig{ + Client: clienToUse, + BaseContext: ctx, + Proxier: apiProxy, + Logger: cacheLogger.Named("leasecache"), + }) + if err != nil { + t.Fatal(err) + } + + mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx)) + + mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, nil, true)) + } else { + mux.Handle("/", ProxyHandler(ctx, apiProxyLogger, apiProxy, nil, true)) + } + + server := &http.Server{ + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 5 * time.Minute, + ErrorLog: apiProxyLogger.StandardLogger(nil), + } + go server.Serve(listener) + + // testClient is the client that is used to talk to the agent for proxying/caching behavior. + testClient, err := activeClient.Clone() + if err != nil { + t.Fatal(err) + } + + if err := testClient.SetAddress("http://" + listener.Addr().String()); err != nil { + t.Fatal(err) + } + + // Login via userpass method to derive a managed token. Set that token as the + // testClient's token + resp, err := testClient.Logical().Write("auth/userpass/login/foo", map[string]interface{}{ + "password": "bar", + }) + if err != nil { + t.Fatal(err) + } + testClient.SetToken(resp.Auth.ClientToken) + + cleanup := func() { + // We wait for a tiny bit for things such as agent renewal to exit properly + time.Sleep(50 * time.Millisecond) + + cluster.Cleanup() + os.Setenv(api.EnvVaultAddress, origEnvVaultAddress) + os.Setenv(api.EnvVaultCACert, origEnvVaultCACert) + listener.Close() + } + + return cleanup, clienToUse, testClient, leaseCache +} diff --git a/command/agent/cache/cache_test.go b/command/agent/cache/cache_test.go index f72d1f3b417f..de66f86cc78b 100644 --- a/command/agent/cache/cache_test.go +++ b/command/agent/cache/cache_test.go @@ -8,7 +8,6 @@ import ( "math/rand" "net" "net/http" - "os" "sync" "testing" "time" @@ -17,7 +16,6 @@ import ( "github.com/hashicorp/go-hclog" kv "github.com/hashicorp/vault-plugin-secrets-kv" "github.com/hashicorp/vault/api" - "github.com/hashicorp/vault/builtin/credential/userpass" "github.com/hashicorp/vault/command/agent/cache/cachememdb" "github.com/hashicorp/vault/command/agent/sink/mock" "github.com/hashicorp/vault/helper/namespace" @@ -28,191 +26,6 @@ import ( "github.com/hashicorp/vault/vault" ) -const policyAdmin = ` -path "*" { - capabilities = ["sudo", "create", "read", "update", "delete", "list"] -} -` - -// setupClusterAndAgent is a helper func used to set up a test cluster and -// caching agent against the active node. It returns a cleanup func that should -// be deferred immediately along with two clients, one for direct cluster -// communication and another to talk to the caching agent. -func setupClusterAndAgent(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { - return setupClusterAndAgentCommon(ctx, t, coreConfig, false, true) -} - -// setupClusterAndAgentNoCache is a helper func used to set up a test cluster and -// proxying agent against the active node. It returns a cleanup func that should -// be deferred immediately along with two clients, one for direct cluster -// communication and another to talk to the caching agent. -func setupClusterAndAgentNoCache(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { - return setupClusterAndAgentCommon(ctx, t, coreConfig, false, false) -} - -// setupClusterAndAgentOnStandby is a helper func used to set up a test cluster -// and caching agent against a standby node. It returns a cleanup func that -// should be deferred immediately along with two clients, one for direct cluster -// communication and another to talk to the caching agent. -func setupClusterAndAgentOnStandby(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig) (func(), *api.Client, *api.Client, *LeaseCache) { - return setupClusterAndAgentCommon(ctx, t, coreConfig, true, true) -} - -func setupClusterAndAgentCommon(ctx context.Context, t *testing.T, coreConfig *vault.CoreConfig, onStandby bool, useCache bool) (func(), *api.Client, *api.Client, *LeaseCache) { - t.Helper() - - if ctx == nil { - ctx = context.Background() - } - - // Handle sane defaults - if coreConfig == nil { - coreConfig = &vault.CoreConfig{ - DisableMlock: true, - DisableCache: true, - Logger: logging.NewVaultLogger(hclog.Trace), - } - } - - // Always set up the userpass backend since we use that to generate an admin - // token for the client that will make proxied requests to through the agent. - if coreConfig.CredentialBackends == nil || coreConfig.CredentialBackends["userpass"] == nil { - coreConfig.CredentialBackends = map[string]logical.Factory{ - "userpass": userpass.Factory, - } - } - - // Init new test cluster - cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ - HandlerFunc: vaulthttp.Handler, - }) - cluster.Start() - - cores := cluster.Cores - vault.TestWaitActive(t, cores[0].Core) - - activeClient := cores[0].Client - standbyClient := cores[1].Client - - // clienToUse is the client for the agent to point to. - clienToUse := activeClient - if onStandby { - clienToUse = standbyClient - } - - // Add an admin policy - if err := activeClient.Sys().PutPolicy("admin", policyAdmin); err != nil { - t.Fatal(err) - } - - // Set up the userpass auth backend and an admin user. Used for getting a token - // for the agent later down in this func. - err := activeClient.Sys().EnableAuthWithOptions("userpass", &api.EnableAuthOptions{ - Type: "userpass", - }) - if err != nil { - t.Fatal(err) - } - - _, err = activeClient.Logical().Write("auth/userpass/users/foo", map[string]interface{}{ - "password": "bar", - "policies": []string{"admin"}, - }) - if err != nil { - t.Fatal(err) - } - - // Set up env vars for agent consumption - origEnvVaultAddress := os.Getenv(api.EnvVaultAddress) - os.Setenv(api.EnvVaultAddress, clienToUse.Address()) - - origEnvVaultCACert := os.Getenv(api.EnvVaultCACert) - os.Setenv(api.EnvVaultCACert, fmt.Sprintf("%s/ca_cert.pem", cluster.TempDir)) - - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - apiProxyLogger := logging.NewVaultLogger(hclog.Trace).Named("apiproxy") - - // Create the API proxier - apiProxy, err := NewAPIProxy(&APIProxyConfig{ - Client: clienToUse, - Logger: apiProxyLogger, - }) - if err != nil { - t.Fatal(err) - } - - // Create a muxer and add paths relevant for the lease cache layer and API proxy layer - mux := http.NewServeMux() - - var leaseCache *LeaseCache - if useCache { - cacheLogger := logging.NewVaultLogger(hclog.Trace).Named("cache") - - // Create the lease cache proxier and set its underlying proxier to - // the API proxier. - leaseCache, err = NewLeaseCache(&LeaseCacheConfig{ - Client: clienToUse, - BaseContext: ctx, - Proxier: apiProxy, - Logger: cacheLogger.Named("leasecache"), - }) - if err != nil { - t.Fatal(err) - } - - mux.Handle("/agent/v1/cache-clear", leaseCache.HandleCacheClear(ctx)) - - mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, nil, true)) - } else { - mux.Handle("/", ProxyHandler(ctx, apiProxyLogger, apiProxy)) - } - - server := &http.Server{ - Handler: mux, - ReadHeaderTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - IdleTimeout: 5 * time.Minute, - ErrorLog: apiProxyLogger.StandardLogger(nil), - } - go server.Serve(listener) - - // testClient is the client that is used to talk to the agent for proxying/caching behavior. - testClient, err := activeClient.Clone() - if err != nil { - t.Fatal(err) - } - - if err := testClient.SetAddress("http://" + listener.Addr().String()); err != nil { - t.Fatal(err) - } - - // Login via userpass method to derive a managed token. Set that token as the - // testClient's token - resp, err := testClient.Logical().Write("auth/userpass/login/foo", map[string]interface{}{ - "password": "bar", - }) - if err != nil { - t.Fatal(err) - } - testClient.SetToken(resp.Auth.ClientToken) - - cleanup := func() { - // We wait for a tiny bit for things such as agent renewal to exit properly - time.Sleep(50 * time.Millisecond) - - cluster.Cleanup() - os.Setenv(api.EnvVaultAddress, origEnvVaultAddress) - os.Setenv(api.EnvVaultCACert, origEnvVaultCACert) - listener.Close() - } - - return cleanup, clienToUse, testClient, leaseCache -} - func tokenRevocationValidation(t *testing.T, sampleSpace map[string]string, expected map[string]string, leaseCache *LeaseCache) { t.Helper() for val, valType := range sampleSpace { @@ -265,7 +78,7 @@ func TestCache_AutoAuthTokenStripping(t *testing.T) { mux := http.NewServeMux() mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) + mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink("testid"), true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, @@ -354,7 +167,7 @@ func TestCache_AutoAuthClientTokenProxyStripping(t *testing.T) { mux := http.NewServeMux() // mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) - mux.Handle("/", CachingProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) + mux.Handle("/", ProxyHandler(ctx, cacheLogger, leaseCache, mock.NewSink(realToken), false)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, diff --git a/command/agent/cache/handler.go b/command/agent/cache/handler.go index a93f2f83cf0f..e634174c61ae 100644 --- a/command/agent/cache/handler.go +++ b/command/agent/cache/handler.go @@ -20,61 +20,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func ProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - logger.Info("received request", "method", r.Method, "path", r.URL.Path) - - token := r.Header.Get(consts.AuthHeaderName) - - // Parse and reset body. - reqBody, err := io.ReadAll(r.Body) - if err != nil { - logger.Error("failed to read request body") - logical.RespondError(w, http.StatusInternalServerError, errors.New("failed to read request body")) - return - } - if r.Body != nil { - r.Body.Close() - } - r.Body = io.NopCloser(bytes.NewReader(reqBody)) - req := &SendRequest{ - Token: token, - Request: r, - RequestBody: reqBody, - } - - resp, err := proxier.Send(ctx, req) - if err != nil { - // If this is an api.Response error, don't wrap the response. - if resp != nil && resp.Response.Error() != nil { - copyHeader(w.Header(), resp.Response.Header) - w.WriteHeader(resp.Response.StatusCode) - io.Copy(w, resp.Response.Body) - metrics.IncrCounter([]string{"agent", "proxy", "client_error"}, 1) - } else { - metrics.IncrCounter([]string{"agent", "proxy", "error"}, 1) - logical.RespondError(w, http.StatusInternalServerError, fmt.Errorf("failed to get the response: %w", err)) - } - return - } - - if err != nil { - logical.RespondError(w, http.StatusInternalServerError, fmt.Errorf("failed to process token lookup response: %w", err)) - return - } - - defer resp.Response.Body.Close() - - // Set headers - setHeaders(w, resp) - - // Set response body - io.Copy(w, resp.Response.Body) - return - }) -} - -func CachingProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { +func ProxyHandler(ctx context.Context, logger hclog.Logger, proxier Proxier, inmemSink sink.Sink, proxyVaultToken bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("received request", "method", r.Method, "path", r.URL.Path) diff --git a/command/agent/cache/proxy.go b/command/agent/cache/proxy.go index ef7a83711f37..af9267ba01cc 100644 --- a/command/agent/cache/proxy.go +++ b/command/agent/cache/proxy.go @@ -3,7 +3,7 @@ package cache import ( "bytes" "context" - "io/ioutil" + "io" "net/http" "time" @@ -59,7 +59,7 @@ func NewSendResponse(apiResponse *api.Response, responseBody []byte) (*SendRespo case len(responseBody) > 0: resp.ResponseBody = responseBody case apiResponse.Body != nil: - respBody, err := ioutil.ReadAll(apiResponse.Body) + respBody, err := io.ReadAll(apiResponse.Body) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func NewSendResponse(apiResponse *api.Response, responseBody []byte) (*SendRespo apiResponse.Body.Close() // Re-set the response body after reading from the Reader - apiResponse.Body = ioutil.NopCloser(bytes.NewReader(respBody)) + apiResponse.Body = io.NopCloser(bytes.NewReader(respBody)) resp.ResponseBody = respBody } diff --git a/command/agent/cache_end_to_end_test.go b/command/agent/cache_end_to_end_test.go index 8a3400cea8c1..6337c918a698 100644 --- a/command/agent/cache_end_to_end_test.go +++ b/command/agent/cache_end_to_end_test.go @@ -315,7 +315,7 @@ func TestCache_UsingAutoAuthToken(t *testing.T) { mux.Handle(consts.AgentPathCacheClear, leaseCache.HandleCacheClear(ctx)) // Passing a non-nil inmemsink tells the agent to use the auto-auth token - mux.Handle("/", cache.CachingProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, true)) + mux.Handle("/", cache.ProxyHandler(ctx, cacheLogger, leaseCache, inmemSink, true)) server := &http.Server{ Handler: mux, ReadHeaderTimeout: 10 * time.Second, diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 745a804ebd84..820a37c8b331 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -19,13 +19,14 @@ import ( "github.com/mitchellh/mapstructure" ) -// Config is the configuration for the vault server. +// Config is the configuration for Vault Agent. type Config struct { *configutil.SharedConfig `hcl:"-"` AutoAuth *AutoAuth `hcl:"auto_auth"` ExitAfterAuth bool `hcl:"exit_after_auth"` Cache *Cache `hcl:"cache"` + APIProxy *APIProxy `hcl:"api_proxy""` Vault *Vault `hcl:"vault"` TemplateConfig *TemplateConfig `hcl:"template_config"` Templates []*ctconfig.TemplateConfig `hcl:"templates"` @@ -88,6 +89,15 @@ type transportDialer interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) } +// APIProxy contains any configuration needed for proxy mode +type APIProxy struct { + UseAutoAuthTokenRaw interface{} `hcl:"use_auto_auth_token"` + UseAutoAuthToken bool `hcl:"-"` + ForceAutoAuthToken bool `hcl:"-"` + EnforceConsistency string `hcl:"enforce_consistency"` + WhenInconsistent string `hcl:"when_inconsistent"` +} + // Cache contains any configuration needed for Cache mode type Cache struct { UseAutoAuthTokenRaw interface{} `hcl:"use_auto_auth_token"` @@ -222,6 +232,20 @@ func LoadConfig(path string) (*Config, error) { return nil, fmt.Errorf("error parsing 'cache':%w", err) } + if err := parseAPIProxy(result, list); err != nil { + return nil, fmt.Errorf("error parsing 'api_proxy':%w", err) + } + + if result.APIProxy != nil && result.Cache != nil { + if result.Cache.UseAutoAuthTokenRaw != nil { + if result.APIProxy.UseAutoAuthTokenRaw != nil { + return nil, fmt.Errorf("use_auto_auth_token defined in both api_proxy and cache config. Please remove this configuration from the cache block") + } else { + result.APIProxy.ForceAutoAuthToken = result.Cache.ForceAutoAuthToken + } + } + } + if err := parseTemplateConfig(result, list); err != nil { return nil, fmt.Errorf("error parsing 'template_config': %w", err) } @@ -230,6 +254,13 @@ func LoadConfig(path string) (*Config, error) { return nil, fmt.Errorf("error parsing 'template': %w", err) } + if result.Cache != nil && result.APIProxy == nil && len(result.Listeners) > 0 { + result.APIProxy = &APIProxy{ + UseAutoAuthToken: result.Cache.UseAutoAuthToken, + ForceAutoAuthToken: result.Cache.ForceAutoAuthToken, + } + } + if result.Cache != nil { if len(result.Listeners) < 1 && len(result.Templates) < 1 { return nil, fmt.Errorf("enabling the cache requires at least 1 template or 1 listener to be defined") @@ -239,17 +270,32 @@ func LoadConfig(path string) (*Config, error) { if result.AutoAuth == nil { return nil, fmt.Errorf("cache.use_auto_auth_token is true but auto_auth not configured") } - if result.AutoAuth.Method.WrapTTL > 0 { + if result.AutoAuth != nil && result.AutoAuth.Method != nil && result.AutoAuth.Method.WrapTTL > 0 { return nil, fmt.Errorf("cache.use_auto_auth_token is true and auto_auth uses wrapping") } } } + if result.APIProxy != nil { + if len(result.Listeners) < 1 { + return nil, fmt.Errorf("configuring the api_proxy requires at least 1 listener to be defined") + } + + if result.APIProxy.UseAutoAuthToken { + if result.AutoAuth == nil { + return nil, fmt.Errorf("api_proxy.use_auto_auth_token is true but auto_auth not configured") + } + if result.AutoAuth != nil && result.AutoAuth.Method != nil && result.AutoAuth.Method.WrapTTL > 0 { + return nil, fmt.Errorf("api_proxy.use_auto_auth_token is true and auto_auth uses wrapping") + } + } + } + if result.AutoAuth != nil { if len(result.AutoAuth.Sinks) == 0 && - (result.Cache == nil || !result.Cache.UseAutoAuthToken) && + (result.APIProxy == nil || !result.APIProxy.UseAutoAuthToken) && len(result.Templates) == 0 { - return nil, fmt.Errorf("auto_auth requires at least one sink or at least one template or cache.use_auto_auth_token=true") + return nil, fmt.Errorf("auto_auth requires at least one sink or at least one template or api_proxy.use_auto_auth_token=true") } } @@ -386,6 +432,50 @@ func parseRetry(result *Config, list *ast.ObjectList) error { return nil } +func parseAPIProxy(result *Config, list *ast.ObjectList) error { + name := "api_proxy" + + apiProxyList := list.Filter(name) + if len(apiProxyList.Items) == 0 { + return nil + } + + if len(apiProxyList.Items) > 1 { + return fmt.Errorf("one and only one %q block is required", name) + } + + item := apiProxyList.Items[0] + + var apiProxy APIProxy + err := hcl.DecodeObject(&apiProxy, item.Val) + if err != nil { + return err + } + + if apiProxy.UseAutoAuthTokenRaw != nil { + apiProxy.UseAutoAuthToken, err = parseutil.ParseBool(apiProxy.UseAutoAuthTokenRaw) + if err != nil { + // Could be a value of "force" instead of "true"/"false" + switch apiProxy.UseAutoAuthTokenRaw.(type) { + case string: + v := apiProxy.UseAutoAuthTokenRaw.(string) + + if !strings.EqualFold(v, "force") { + return fmt.Errorf("value of 'use_auto_auth_token' can be either true/false/force, %q is an invalid option", apiProxy.UseAutoAuthTokenRaw) + } + apiProxy.UseAutoAuthToken = true + apiProxy.ForceAutoAuthToken = true + + default: + return err + } + } + } + result.APIProxy = &apiProxy + + return nil +} + func parseCache(result *Config, list *ast.ObjectList) error { name := "cache" diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 6a95284b5189..ba601e56b6a1 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -69,6 +69,10 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { }, }, }, + APIProxy: &APIProxy{ + UseAutoAuthToken: true, + ForceAutoAuthToken: false, + }, Cache: &Cache{ UseAutoAuthToken: true, UseAutoAuthTokenRaw: true, @@ -394,7 +398,8 @@ func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) { } expected := &Config{ - Cache: &Cache{}, + APIProxy: &APIProxy{}, + Cache: &Cache{}, SharedConfig: &configutil.SharedConfig{ PidFile: "./pidfile", Listeners: []*configutil.Listener{ @@ -467,6 +472,13 @@ func TestLoadConfigFile_Bad_AgentCache_AutoAuth_Method_wrapping(t *testing.T) { } } +func TestLoadConfigFile_Bad_APIProxy_And_Cache_Same_Config(t *testing.T) { + _, err := LoadConfig("./test-fixtures/bad-config-api_proxy-cache.hcl") + if err == nil { + t.Fatal("LoadConfig should return an error when cache and api_proxy try and configure the same value") + } +} + func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-no-sink.hcl") if err != nil { @@ -493,6 +505,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { }, }, }, + APIProxy: &APIProxy{ + UseAutoAuthToken: true, + ForceAutoAuthToken: false, + }, Cache: &Cache{ UseAutoAuthToken: true, UseAutoAuthTokenRaw: true, @@ -537,6 +553,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_Force(t *testing.T) { }, }, }, + APIProxy: &APIProxy{ + UseAutoAuthToken: true, + ForceAutoAuthToken: true, + }, Cache: &Cache{ UseAutoAuthToken: true, UseAutoAuthTokenRaw: "force", @@ -581,6 +601,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_True(t *testing.T) { }, }, }, + APIProxy: &APIProxy{ + UseAutoAuthToken: true, + ForceAutoAuthToken: false, + }, Cache: &Cache{ UseAutoAuthToken: true, UseAutoAuthTokenRaw: "true", @@ -599,6 +623,52 @@ func TestLoadConfigFile_AgentCache_AutoAuth_True(t *testing.T) { } } +func TestLoadConfigFile_Agent_AutoAuth_APIProxyAllConfig(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + Listeners: []*configutil.Listener{ + { + Type: "tcp", + Address: "127.0.0.1:8300", + TLSDisable: true, + }, + }, + PidFile: "./pidfile", + }, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + }, + APIProxy: &APIProxy{ + UseAutoAuthToken: true, + UseAutoAuthTokenRaw: "force", + ForceAutoAuthToken: true, + EnforceConsistency: "always", + WhenInconsistent: "forward", + }, + Vault: &Vault{ + Retry: &Retry{ + NumRetries: 12, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + func TestLoadConfigFile_AgentCache_AutoAuth_False(t *testing.T) { config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-false.hcl") if err != nil { @@ -636,6 +706,10 @@ func TestLoadConfigFile_AgentCache_AutoAuth_False(t *testing.T) { }, }, }, + APIProxy: &APIProxy{ + UseAutoAuthToken: false, + ForceAutoAuthToken: false, + }, Cache: &Cache{ UseAutoAuthToken: false, UseAutoAuthTokenRaw: "false", @@ -661,6 +735,7 @@ func TestLoadConfigFile_AgentCache_Persist(t *testing.T) { } expected := &Config{ + APIProxy: &APIProxy{}, Cache: &Cache{ Persist: &Persist{ Type: "kubernetes", @@ -1075,6 +1150,7 @@ func TestLoadConfigFile_EnforceConsistency(t *testing.T) { }, PidFile: "", }, + APIProxy: &APIProxy{}, Cache: &Cache{ EnforceConsistency: "always", WhenInconsistent: "retry", @@ -1092,6 +1168,40 @@ func TestLoadConfigFile_EnforceConsistency(t *testing.T) { } } +func TestLoadConfigFile_EnforceConsistency_APIProxy(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-consistency-apiproxy.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + Listeners: []*configutil.Listener{ + { + Type: "tcp", + Address: "127.0.0.1:8300", + TLSDisable: true, + }, + }, + PidFile: "", + }, + APIProxy: &APIProxy{ + EnforceConsistency: "always", + WhenInconsistent: "retry", + }, + Vault: &Vault{ + Retry: &Retry{ + NumRetries: 12, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + func TestLoadConfigFile_Disable_Idle_Conns_All(t *testing.T) { config, err := LoadConfig("./test-fixtures/config-disable-idle-connections-all.hcl") if err != nil { diff --git a/command/agent/config/test-fixtures/bad-config-api_proxy-cache.hcl b/command/agent/config/test-fixtures/bad-config-api_proxy-cache.hcl new file mode 100644 index 000000000000..ae79293b483a --- /dev/null +++ b/command/agent/config/test-fixtures/bad-config-api_proxy-cache.hcl @@ -0,0 +1,23 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + config = { + role = "foobar" + } + } +} + +cache { + use_auto_auth_token = true +} + +api_proxy { + use_auto_auth_token = "force" +} + +listener "tcp" { + address = "127.0.0.1:8300" + tls_disable = true +} diff --git a/command/agent/config/test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl b/command/agent/config/test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl new file mode 100644 index 000000000000..b486418eedef --- /dev/null +++ b/command/agent/config/test-fixtures/config-api_proxy-auto_auth-all-api_proxy-config.hcl @@ -0,0 +1,21 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + config = { + role = "foobar" + } + } +} + +api_proxy { + use_auto_auth_token = "force" + enforce_consistency = "always" + when_inconsistent = "forward" +} + +listener "tcp" { + address = "127.0.0.1:8300" + tls_disable = true +} diff --git a/command/agent/config/test-fixtures/config-cache-auto_auth-true.hcl b/command/agent/config/test-fixtures/config-cache-auto_auth-true.hcl index ccadc501d07c..5a46d1b93c5c 100644 --- a/command/agent/config/test-fixtures/config-cache-auto_auth-true.hcl +++ b/command/agent/config/test-fixtures/config-cache-auto_auth-true.hcl @@ -11,10 +11,10 @@ auto_auth { cache { use_auto_auth_token = "true" + force_auto_auth_token = false } listener "tcp" { address = "127.0.0.1:8300" tls_disable = true } - diff --git a/command/agent/config/test-fixtures/config-consistency-apiproxy.hcl b/command/agent/config/test-fixtures/config-consistency-apiproxy.hcl new file mode 100644 index 000000000000..d116964a1b50 --- /dev/null +++ b/command/agent/config/test-fixtures/config-consistency-apiproxy.hcl @@ -0,0 +1,9 @@ +api_proxy { + enforce_consistency = "always" + when_inconsistent = "retry" +} + +listener "tcp" { + address = "127.0.0.1:8300" + tls_disable = true +} From 30c296cbebe8e5e84857aa8547005ef88500e5d2 Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Thu, 1 Dec 2022 13:50:47 -0500 Subject: [PATCH 07/10] VAULT-11510 First pass at API Proxy docs --- website/content/docs/agent/apiproxy.mdx | 87 +++++++++++++++++++ website/content/docs/agent/caching/index.mdx | 66 +++++--------- website/content/docs/agent/index.mdx | 27 ++++-- .../content/docs/enterprise/consistency.mdx | 4 +- 4 files changed, 133 insertions(+), 51 deletions(-) create mode 100644 website/content/docs/agent/apiproxy.mdx diff --git a/website/content/docs/agent/apiproxy.mdx b/website/content/docs/agent/apiproxy.mdx new file mode 100644 index 000000000000..6ac3c482abed --- /dev/null +++ b/website/content/docs/agent/apiproxy.mdx @@ -0,0 +1,87 @@ +--- +layout: docs +page_title: Vault Agent API Proxy +description: >- + Vault Agent's API Proxy functionality allows you to use Vault Agent's API as a proxy + for Vault's API. +--- + +# Vault Agent API Proxy + +Vault Agent's API Proxy functionality allows you to use Vault Agent's API as a proxy +for Vault's API. + +## Functionality + +The [`listener` stanza](/docs/agent#listener-stanza) for Vault Agent configures a listener for Vault Agent. If +its `role` is not set to `metrics_only`, it will act as a proxy for the Vault server that +has been configured in the `vault` stanza of Vault Agent. This enables access to the Vault +API from the Agent API, and can be configured to optionally allow or force the automatic use of +the Auto-Auth token for these requests, as described below. + +If a `listener` has been configured alongside a `cache` stanza, the API Proxy will +go through and utilize the cache subsystem. See the [caching docs](/docs/agent/caching) +for more information on caching. + +## Using Auto-Auth Token + +Vault Agent allows for easy authentication to Vault in a wide variety of +environments using [Auto-Auth](/docs/agent/autoauth). By setting the +`use_auto_auth_token` (see below) configuration, clients will not be required +to provide a Vault token to the requests made to the Agent. When this +configuration is set, if the request doesn't already bear a token, then the +auto-auth token will be used to forward the request to the Vault server. This +configuration will be overridden if the request already has a token attached, +in which case, the token present in the request will be used to forward the +request to the Vault server. + +## Forcing Auto-Auth Token + +Vault Agent can be configured to force the use of the auto-auth token by using +the value `force` for the `use_auto_auth_token` option. This configuration +overrides the default behavior described above in [Using Auto-Auth +Token](/docs/agent/apiproxy#using-auto-auth-token), and instead ignores any +existing Vault token in the request and instead uses the auto-auth token. + + +## Configuration (`api_proxy`) + +The top level `api_proxy` block has the following configuration entries: + +- `use_auto_auth_token (bool/string: false)` - If set, the requests made to Agent +without a Vault token will be forwarded to the Vault server with the +auto-auth token attached. If the requests already bear a token, this +configuration will be overridden and the token in the request will be used to +forward the request to the Vault server. If set to `"force"` Agent will use the +auto-auth token, overwriting the attached Vault token if set. + +The following two `api_proxy` options are only useful when talking to a Vault +Enterprise cluster, and are documented as part of its +[Eventual Consistency](/docs/enterprise/consistency#vault-agent-and-consistency-headers) +page. + +- `enforce_consistency` `(string: "never")` - Set to one of `"always"` +or `"never"`. + +- `when_inconsistent` `(string: optional)` - Set to one of `"fail"`, `"retry"`, +or `"forward"`. + +### Example Configuration + +Here is an example of a `listener` configuration alongside `api_proxy` configuration to force the use of the auto_auth token +and enforce consistency. + +```hcl +# Other Vault Agent configuration blocks +# ... + +api_proxy { + use_auto_auth_token = "force" + enforce_consistency = "always" +} + +listener "tcp" { + address = "127.0.0.1:8100" + tls_disable = true +} +``` diff --git a/website/content/docs/agent/caching/index.mdx b/website/content/docs/agent/caching/index.mdx index 6cc96f17bca8..e5d0f0458113 100644 --- a/website/content/docs/agent/caching/index.mdx +++ b/website/content/docs/agent/caching/index.mdx @@ -36,26 +36,6 @@ specific scenarios. that are issued using the tokens managed by the agent, will be cached and its renewals are taken care of. -## Using Auto-Auth Token - -Vault Agent allows for easy authentication to Vault in a wide variety of -environments using [Auto-Auth](/docs/agent/autoauth). By setting the -`use_auto_auth_token` (see below) configuration, clients will not be required -to provide a Vault token to the requests made to the agent. When this -configuration is set, if the request doesn't already bear a token, then the -auto-auth token will be used to forward the request to the Vault server. This -configuration will be overridden if the request already has a token attached, -in which case, the token present in the request will be used to forward the -request to the Vault server. - -## Forcing Auto-Auth Token - -Vault Agent can be configured to force the use of the auto-auth token by using -the value `force` for the `use_auto_auth_token` option. This configuration -overrides the default behavior described above in [Using Auto-Auth -Token](/docs/agent/caching#using-auto-auth-token), and instead ignores any -existing Vault token in the request and instead uses the auto-auth token. - ## Persistent Cache Vault Agent can restore tokens and leases from a persistent cache file created @@ -160,7 +140,7 @@ but will do nothing (there is no cache to clear) and will return a `200` respons ```json { "type": "token", - "value": "s.rlNjegSKykWcplOkwsjd8bP9" + "value": "hvs.rlNjegSKykWcplOkwsjd8bP9" } ``` @@ -175,34 +155,24 @@ $ curl \ ## Configuration (`cache`) -The top level `cache` block has the following configuration entries: - -- `use_auto_auth_token (bool/string: false)` - If set, the requests made to agent - without a Vault token will be forwarded to the Vault server with the - auto-auth token attached. If the requests already bear a token, this - configuration will be overridden and the token in the request will be used to - forward the request to the Vault server. If set to `"force"` Agent will use the - auto-auth token, overwriting the attached Vault token if set. +The presence of the top level `cache` block in any way (including an empty `cache` block) will enable the cache. +The top level `cache` block has the following configuration entry: - `persist` `(object: optional)` - Configuration for the persistent cache. -The following two `cache` options are only useful when talking to a Vault -Enterprise cluster, and are documented as part of its -[Eventual Consistency](/docs/enterprise/consistency#vault-agent-and-consistency-headers) -page. - -- `enforce_consistency` `(string: "never")` - Set to one of `"always"` - or `"never"`. - -- `when_inconsistent` `(string: optional)` - Set to one of `"fail"`, `"retry"`, - or `"forward"`. +The `cache` block also supports the `use_auto_auth_token`, `enforce_consistency`, and +`when_inconsistent` configuration values of the `api_proxy` block +[described in the API Proxy documentation](/docs/agent/apiproxy#configuration-api_proxy) for +legacy reasons. This configuration cannot be specified alongside `api_proxy` equivalents, +should not be preferred over configuring these values in the `api_proxy` block, +and should be considered legacy. -> **Note:** When the `cache` block is defined, at least one [template][agent-template] or [listener][agent-listener] must also be defined in the config, otherwise there is no way to utilize the cache. [agent-template]: /docs/agent/template -[agent-listener]: /docs/agent/caching#configuration-listener +[agent-listener]: /docs/agent#listener-stanza ### Configuration (Persist) @@ -225,8 +195,9 @@ These are common configuration values that live within the `persist` block: - `listener` `(array of objects: required)` - Configuration for the listeners. -There can be one or more `listener` blocks at the top level. These configuration -values are common to both `tcp` and `unix` listener blocks. Blocks of type +There can be one or more `listener` blocks at the top level. Adding a listener enables +the [API Proxy](/docs/agent/apiproxy) and enables the API proxy to use the cache. +These configuration values are common to both `tcp` and `unix` listener blocks. Blocks of type `tcp` support the standard `tcp` [listener](/docs/configuration/listener/tcp) options. Additionally, the `role` string option is available as part of the top level of the `listener` block, which can be configured to `metrics_only` to serve only metrics, @@ -252,14 +223,21 @@ or the default role, `default`, which serves everything (including metrics). ### Example Configuration -Here is an example of a cache configuration alongside a regular listener, and a listener that only serves metrics. +Here is an example of a cache configuration with the optional `persist` block, +alongside a regular listener, and a listener that only serves metrics. ```hcl # Other Vault Agent configuration blocks # ... cache { - use_auto_auth_token = true + persist = { + type = "kubernetes" + path = "/vault/agent-cache/" + keep_after_import = true + exit_on_err = true + service_account_token_file = "/tmp/serviceaccount/token" + } } listener "tcp" { diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index 8be0f799931d..51a919a9dc93 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -85,6 +85,8 @@ Vault Agent is a client daemon that provides the following features: - [Auto-Auth][autoauth] - Automatically authenticate to Vault and manage the token renewal process for locally-retrieved dynamic secrets. +- [API Proxy][apiproxy] - Allows Vault Agent to act as a proxy for Vault's API, + optionally using (or forcing the use of) the Auto-Auth token. - [Caching][caching] - Allows client-side caching of responses containing newly created tokens and responses containing leased secrets generated off of these newly created tokens. The agent also manages the renewals of the cached tokens and leases. @@ -101,6 +103,16 @@ for information. Auto-Auth functionality takes place within an `auto_auth` configuration stanza. +## API Proxy + +Vault Agent can act as an API proxy for Vault, allowing you to talk to Vault's +APIs a listener defined for Agent. It can be configured to optionally allow or force the automatic use of +the Auto-Auth token for these requests. Please see the [API Proxy docs][apiproxy] +for more information. + +API Proxy functionality takes place within a defined `listener`, and its behaviour can be configured with an +`api_proxy` configuration stanza. + ## Caching Vault Agent allows client-side caching of responses containing newly created tokens @@ -168,6 +180,8 @@ These are the currently-available general configuration option: - `auto_auth` ([auto_auth][autoauth]: ) - Specifies the method and other options used for Auto-Auth functionality. +- `api_proxy` ([api_proxy][apiproxy]: ) - Specifies options used for API Proxy functionality. + - `cache` ([cache][caching]: ) - Specifies options used for Caching functionality. - `listener` ([listener][listener]: ) - Specifies the addresses and ports on which the Agent will respond to requests. @@ -211,7 +225,7 @@ These are the currently-available general configuration option: ### vault Stanza -There can at most be one top level `vault` block and it has the following +There can at most be one top level `vault` block, and it has the following configuration entries: - `address` `(string: )` - The address of the Vault server to @@ -251,7 +265,7 @@ configuration entries: The `vault` stanza may contain a `retry` stanza that controls how failing Vault requests are handled, whether these requests are issued in order to render -templates, or are proxied requests coming from the proxy cache subsystem. +templates, or are proxied requests coming from the api proxy subsystem. Auto-auth, however, has its own notion of retrying and is not affected by this section. @@ -288,8 +302,8 @@ to address in the future. Vault Agent supports one or more [listener][listener_main] stanzas. Listeners can be configured with or without [caching][caching], but will use the cache if it -has been configured. In addition to the standard listener configuration, -an Agent's listener configuration also supports the following: +has been configured, and will enable the [API proxy][apiproxy]. In addition to the standard +listener configuration, an Agent's listener configuration also supports the following: - `require_request_header` `(bool: false)` - Require that all incoming HTTP requests on this listener must have an `X-Vault-Request: true` header entry. @@ -387,7 +401,9 @@ auto_auth { } } -cache { +cache {} + +api_proxy { use_auto_auth_token = true } @@ -419,6 +435,7 @@ template { [vault]: /docs/agent#vault-stanza [autoauth]: /docs/agent/autoauth [caching]: /docs/agent/caching +[apiproxy]: /docs/agent/apiproxy [persistent-cache]: /docs/agent/caching/persistent-caches [template]: /docs/agent/template [template-config]: /docs/agent/template#template-configurations diff --git a/website/content/docs/enterprise/consistency.mdx b/website/content/docs/enterprise/consistency.mdx index 11cb65fb9515..a0be208daf30 100644 --- a/website/content/docs/enterprise/consistency.mdx +++ b/website/content/docs/enterprise/consistency.mdx @@ -175,8 +175,8 @@ to build equivalent functionality into their client library. ### Vault Agent and consistency headers -Vault Agent Caching will proxy incoming requests to Vault. There is -new Agent configuration available in the `cache` stanza that allows making use +When configured, the [Vault Agent API Proxy](/docs/agent/apiproxy) will proxy incoming requests to Vault. There is +Agent configuration available in the `api_proxy` stanza that allows making use of some of the above mitigations without modifying clients. By setting `enforce_consistency="always"`, Agent will always provide From 6f7577ee936729e571d110f19bbbf56a3d337ddd Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Thu, 1 Dec 2022 14:17:08 -0500 Subject: [PATCH 08/10] VAULT-11510 nav data --- website/data/docs-nav-data.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 41374eced89f..404210ebe33d 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -894,6 +894,10 @@ } ] }, + { + "title": "API Proxy", + "path": "agent/apiproxy" + }, { "title": "Caching", "routes": [ From a0e39edca9db5aeb5a5249ddc268447b10a7cc93 Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Mon, 5 Dec 2022 08:53:59 -0500 Subject: [PATCH 09/10] VAULT-11510 typo --- command/agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command/agent.go b/command/agent.go index e50c9610733f..f9ebf1c7c187 100644 --- a/command/agent.go +++ b/command/agent.go @@ -462,14 +462,14 @@ func (c *AgentCommand) Run(args []string) int { switch config.Cache.WhenInconsistent { case "retry": if whenInconsistent != cache.WhenInconsistentFail { - c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") + c.UI.Error("when_inconsistent configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") return 1 } else { whenInconsistent = cache.WhenInconsistentRetry } case "forward": if whenInconsistent != cache.WhenInconsistentFail { - c.UI.Error("enforce_consistency configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") + c.UI.Error("when_inconsistent configured in both api_proxy and cache blocks. Please remove this configuration from the cache block.") return 1 } else { whenInconsistent = cache.WhenInconsistentForward From ceca2ffd0f6c9cffe0bbf761802a2f8b9767a12d Mon Sep 17 00:00:00 2001 From: VioletHynes Date: Mon, 5 Dec 2022 09:18:16 -0500 Subject: [PATCH 10/10] VAULT-11510 docs update --- website/content/docs/agent/apiproxy.mdx | 10 +++++----- website/content/docs/agent/caching/index.mdx | 12 ++++++++---- website/content/docs/agent/index.mdx | 10 ++++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/website/content/docs/agent/apiproxy.mdx b/website/content/docs/agent/apiproxy.mdx index 6ac3c482abed..7e763ecb310b 100644 --- a/website/content/docs/agent/apiproxy.mdx +++ b/website/content/docs/agent/apiproxy.mdx @@ -15,13 +15,13 @@ for Vault's API. The [`listener` stanza](/docs/agent#listener-stanza) for Vault Agent configures a listener for Vault Agent. If its `role` is not set to `metrics_only`, it will act as a proxy for the Vault server that -has been configured in the `vault` stanza of Vault Agent. This enables access to the Vault +has been configured in the [`vault` stanza](/docs/agent#vault-stanza) stanza of Vault Agent. This enables access to the Vault API from the Agent API, and can be configured to optionally allow or force the automatic use of the Auto-Auth token for these requests, as described below. If a `listener` has been configured alongside a `cache` stanza, the API Proxy will -go through and utilize the cache subsystem. See the [caching docs](/docs/agent/caching) -for more information on caching. +first attempt to utilize the cache subsystem for qualifying requests, before forwarding the +request to Vault. See the [caching docs](/docs/agent/caching) for more information on caching. ## Using Auto-Auth Token @@ -48,14 +48,14 @@ existing Vault token in the request and instead uses the auto-auth token. The top level `api_proxy` block has the following configuration entries: -- `use_auto_auth_token (bool/string: false)` - If set, the requests made to Agent +- `use_auto_auth_token` `(bool/string: false)` - If set, the requests made to Agent without a Vault token will be forwarded to the Vault server with the auto-auth token attached. If the requests already bear a token, this configuration will be overridden and the token in the request will be used to forward the request to the Vault server. If set to `"force"` Agent will use the auto-auth token, overwriting the attached Vault token if set. -The following two `api_proxy` options are only useful when talking to a Vault +The following two `api_proxy` options are only useful when making requests to a Vault Enterprise cluster, and are documented as part of its [Eventual Consistency](/docs/enterprise/consistency#vault-agent-and-consistency-headers) page. diff --git a/website/content/docs/agent/caching/index.mdx b/website/content/docs/agent/caching/index.mdx index e5d0f0458113..0247e678fbbd 100644 --- a/website/content/docs/agent/caching/index.mdx +++ b/website/content/docs/agent/caching/index.mdx @@ -162,10 +162,10 @@ The top level `cache` block has the following configuration entry: The `cache` block also supports the `use_auto_auth_token`, `enforce_consistency`, and `when_inconsistent` configuration values of the `api_proxy` block -[described in the API Proxy documentation](/docs/agent/apiproxy#configuration-api_proxy) for -legacy reasons. This configuration cannot be specified alongside `api_proxy` equivalents, +[described in the API Proxy documentation](/docs/agent/apiproxy#configuration-api_proxy) only to +maintain backwards compatibility. This configuration **cannot** be specified alongside `api_proxy` equivalents, should not be preferred over configuring these values in the `api_proxy` block, -and should be considered legacy. +and `api_proxy` should be the preferred place to configure these values. -> **Note:** When the `cache` block is defined, at least one [template][agent-template] or [listener][agent-listener] must also be defined @@ -191,12 +191,16 @@ These are common configuration values that live within the `persist` block: - `exit_on_err` `(bool: optional)` - When set to true, if any errors occur during a persistent cache restore, Vault Agent will exit with an error. Defaults to `true`. +- `service_account_token_file` `(string: optional)` - When `type` is set to `kubernetes`, +this configures the path on disk where the Kubernetes service account token can be found. +Defaults to `/var/run/secrets/kubernetes.io/serviceaccount/token`. + ## Configuration (`listener`) - `listener` `(array of objects: required)` - Configuration for the listeners. There can be one or more `listener` blocks at the top level. Adding a listener enables -the [API Proxy](/docs/agent/apiproxy) and enables the API proxy to use the cache. +the [API Proxy](/docs/agent/apiproxy) and enables the API proxy to use the cache, if configured. These configuration values are common to both `tcp` and `unix` listener blocks. Blocks of type `tcp` support the standard `tcp` [listener](/docs/configuration/listener/tcp) options. Additionally, the `role` string option is available as part of the top level diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index 51a919a9dc93..c28f3da47bd1 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -106,12 +106,12 @@ Auto-Auth functionality takes place within an `auto_auth` configuration stanza. ## API Proxy Vault Agent can act as an API proxy for Vault, allowing you to talk to Vault's -APIs a listener defined for Agent. It can be configured to optionally allow or force the automatic use of +API via a listener defined for Agent. It can be configured to optionally allow or force the automatic use of the Auto-Auth token for these requests. Please see the [API Proxy docs][apiproxy] for more information. API Proxy functionality takes place within a defined `listener`, and its behaviour can be configured with an -`api_proxy` configuration stanza. +[`api_proxy` stanza](/docs/agent/apiproxy#configuration-api_proxy). ## Caching @@ -174,7 +174,7 @@ See the [caching](/docs/agent/caching#api) page for details on the cache API. ### Configuration File Options -These are the currently-available general configuration option: +These are the currently-available general configuration options: - `vault` ([vault][vault]: ) - Specifies the remote Vault server the Agent connects to. @@ -401,7 +401,9 @@ auto_auth { } } -cache {} +cache { + // An empty cache stanza still enables caching +} api_proxy { use_auto_auth_token = true