Support for vault namespaces

client/allocrunner/taskrunner/template/template.go

@@ -645,6 +645,9 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 		conf.Vault.Address = &cc.VaultConfig.Addr
 		conf.Vault.Token = &config.VaultToken
 		conf.Vault.Grace = helper.TimeToPtr(vaultGrace)
+		if config.ClientConfig.VaultConfig.Namespace != "" {
+			conf.Vault.Namespace = &config.ClientConfig.VaultConfig.Namespace
+		}
 
 		if strings.HasPrefix(cc.VaultConfig.Addr, "https") || cc.VaultConfig.TLSCertFile != "" {
 			skipVerify := cc.VaultConfig.TLSSkipVerify != nil && *cc.VaultConfig.TLSSkipVerify

client/allocrunner/taskrunner/template/template_test.go

@@ -1326,6 +1326,38 @@ func TestTaskTemplateManager_Config_VaultGrace(t *testing.T) {
 	assert.Equal(10*time.Second, *ctconf.Vault.Grace, "Vault Grace Value")
 }
 
+// TestTaskTemplateManager_Config_VaultNamespace asserts the Vault namespace setting is
+// propagated to consul-template's configuration.
+func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+
+	testNS := "test-namespace"
+	c := config.DefaultConfig()
+	c.Node = mock.Node()
+	c.VaultConfig = &sconfig.VaultConfig{
+		Enabled:       helper.BoolToPtr(true),
+		Addr:          "https://localhost/",
+		TLSServerName: "notlocalhost",
+		Namespace:     testNS,
+	}
+
+	alloc := mock.Alloc()
+	config := &TaskTemplateManagerConfig{
+		ClientConfig: c,
+		VaultToken:   "token",
+		EnvBuilder:   taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region),
+	}
+
+	ctmplMapping, err := parseTemplateConfigs(config)
+	assert.Nil(err, "Parsing Templates")
+
+	ctconf, err := newRunnerConfig(config, ctmplMapping)
+	assert.Nil(err, "Building Runner Config")
+	assert.NotNil(ctconf.Vault.Grace, "Vault Grace Pointer")
+	assert.Equal(testNS, *ctconf.Vault.Namespace, "Vault Namespace Value")
+}
+
 func TestTaskTemplateManager_BlockedEvents(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)

client/vaultclient/vaultclient.go

@@ -163,6 +163,12 @@ func NewVaultClient(config *config.VaultConfig, logger hclog.Logger, tokenDerive
 		"User-Agent": []string{"hashicorp/nomad"},
 	})
 
+	// SetHeaders above will replace all headers, make this call second
+	if config.Namespace != "" {
+		logger.Debug("configuring Vault namespace", "namespace", config.Namespace)
+		client.SetNamespace(config.Namespace)
+	}
+
 	c.client = client
 
 	return c, nil

client/vaultclient/vaultclient_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/testutil"
 	vaultapi "github.com/hashicorp/vault/api"
+	vaultconsts "github.com/hashicorp/vault/helper/consts"
 	"github.com/stretchr/testify/require"
 )
 
@@ -92,6 +93,28 @@ func TestVaultClient_TokenRenewals(t *testing.T) {
 	}
 }
 
+// TestVaultClient_NamespaceSupport tests that the Vault namespace config, if present, will result in the
+// namespace header being set on the created Vault client.
+func TestVaultClient_NamespaceSupport(t *testing.T) {
+	t.Parallel()
+	require := require.New(t)
+	tr := true
+	testNs := "test-namespace"
+
+	logger := testlog.HCLogger(t)
+
+	conf := config.DefaultConfig()
+	conf.VaultConfig.Enabled = &tr
+	conf.VaultConfig.Token = "testvaulttoken"
+	conf.VaultConfig.Namespace = testNs
+	c, err := NewVaultClient(conf.VaultConfig, logger, nil)
+	if err != nil {
+		t.Fatalf("failed to build vault client: %v", err)
+	}
+
+	require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName))
+}
+
 func TestVaultClient_Heap(t *testing.T) {
 	t.Parallel()
 	tr := true

command/agent/command.go

@@ -163,6 +163,7 @@ func (c *Command) readConfig() *Config {
 	}), "vault-allow-unauthenticated", "")
 	flags.StringVar(&cmdConfig.Vault.Token, "vault-token", "", "")
 	flags.StringVar(&cmdConfig.Vault.Addr, "vault-address", "", "")
+	flags.StringVar(&cmdConfig.Vault.Namespace, "vault-namespace", "", "")
 	flags.StringVar(&cmdConfig.Vault.Role, "vault-create-from-role", "", "")
 	flags.StringVar(&cmdConfig.Vault.TLSCaFile, "vault-ca-file", "", "")
 	flags.StringVar(&cmdConfig.Vault.TLSCaPath, "vault-ca-path", "", "")
@@ -260,6 +261,13 @@ func (c *Command) readConfig() *Config {
 		}
 	}
 
+	// Check to see if we should read the Vault namespace from the environment
+	if config.Vault.Namespace == "" {
+		if ns, ok := os.LookupEnv("VAULT_NAMESPACE"); ok {
+			config.Vault.Namespace = ns
+		}
+	}
+
 	// Default the plugin directory to be under that of the data directory if it
 	// isn't explicitly specified.
 	if config.PluginDir == "" && config.DataDir != "" {

command/agent/config_parse.go

@@ -892,6 +892,7 @@ func parseVaultConfig(result **config.VaultConfig, list *ast.ObjectList) error {
 		"tls_server_name",
 		"tls_skip_verify",
 		"token",
+		"namespace",
 	}
 
 	if err := helper.CheckHCLKeys(listVal, valid); err != nil {

e2e/vault/matrix_test.go

@@ -3,6 +3,10 @@ package vault
 var (
 	// versions is the set of Vault versions we test for backwards compatibility
 	versions = []string{
+		"1.1.0",
+		"1.0.3",
+		"1.0.2",
+		"1.0.1",
 		"1.0.0",
 		"0.11.5",
 		"0.11.4",

e2e/vault/vault_test.go

@@ -15,6 +15,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/go-version"
+
 	"github.com/hashicorp/nomad/command/agent"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs/config"
@@ -184,20 +186,20 @@ func TestVaultCompatibility(t *testing.T) {
 	for version, vaultBin := range vaultBinaries {
 		vbin := vaultBin
 		t.Run(version, func(t *testing.T) {
-			testVaultCompatibility(t, vbin)
+			testVaultCompatibility(t, vbin, version)
 		})
 	}
 }
 
 // testVaultCompatibility tests compatibility with the given vault binary
-func testVaultCompatibility(t *testing.T, vault string) {
+func testVaultCompatibility(t *testing.T, vault string, version string) {
 	require := require.New(t)
 
 	// Create a Vault server
 	v := testutil.NewTestVaultFromPath(t, vault)
 	defer v.Stop()
 
-	token := setupVault(t, v.Client)
+	token := setupVault(t, v.Client, version)
 
 	// Create a Nomad agent using the created vault
 	nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
@@ -251,11 +253,32 @@ func testVaultCompatibility(t *testing.T, vault string) {
 
 // setupVault takes the Vault client and creates the required policies and
 // roles. It returns the token that should be used by Nomad
-func setupVault(t *testing.T, client *vapi.Client) string {
+func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
 	// Write the policy
 	sys := client.Sys()
-	if err := sys.PutPolicy("nomad-server", policy); err != nil {
-		t.Fatalf("failed to create policy: %v", err)
+
+	// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
+	// perform this using a raw HTTP request
+	newApi, _ := version.NewVersion("0.9.0")
+	testVersion, err := version.NewVersion(vaultVersion)
+	if err != nil {
+		t.Fatalf("failed to parse test version from '%v': %v", t.Name(), err)
+	}
+	if testVersion.LessThan(newApi) {
+		body := map[string]string{
+			"rules": policy,
+		}
+		request := client.NewRequest("PUT", fmt.Sprintf("/v1/sys/policy/%s", "nomad-server"))
+		if err := request.SetJSONBody(body); err != nil {
+			t.Fatalf("failed to set JSON body on legacy policy creation: %v", err)
+		}
+		if _, err := client.RawRequest(request); err != nil {
+			t.Fatalf("failed to create legacy policy: %v", err)
+		}
+	} else {
+		if err := sys.PutPolicy("nomad-server", policy); err != nil {
+			t.Fatalf("failed to create policy: %v", err)
+		}
 	}
 
 	// Build the role

nomad/structs/config/vault.go

@@ -37,6 +37,10 @@ type VaultConfig struct {
 	// role the token is from.
 	Role string `mapstructure:"create_from_role"`
 
+	// Namespaces sets the Vault namespace used for all calls against the
+	// Vault API. If this is unset, then Nomad does not use Vault namespaces.
+	Namespace string `mapstructure:"namespace"`
+
 	// AllowUnauthenticated allows users to submit jobs requiring Vault tokens
 	// without providing a Vault token proving they have access to these
 	// policies.
@@ -106,6 +110,9 @@ func (a *VaultConfig) Merge(b *VaultConfig) *VaultConfig {
 	if b.Token != "" {
 		result.Token = b.Token
 	}
+	if b.Namespace != "" {
+		result.Namespace = b.Namespace
+	}
 	if b.Role != "" {
 		result.Role = b.Role
 	}

nomad/vault.go

@@ -10,12 +10,13 @@ import (
 	"sync/atomic"
 	"time"
 
-	tomb "gopkg.in/tomb.v2"
+	"gopkg.in/tomb.v2"
 
-	metrics "github.com/armon/go-metrics"
+	"github.com/armon/go-metrics"
 	log "github.com/hashicorp/go-hclog"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	vapi "github.com/hashicorp/vault/api"
+	vaultconsts "github.com/hashicorp/vault/helper/consts"
 
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
@@ -252,6 +253,11 @@ func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVault
 			return nil, err
 		}
 
+		if c.Namespace != "" {
+			logger.Debug("configuring Vault namespace", "namespace", c.Namespace)
+			v.client.SetNamespace(c.Namespace)
+		}
+
 		// Launch the required goroutines
 		v.tomb.Go(wrapNilError(v.establishConnection))
 		v.tomb.Go(wrapNilError(v.revokeDaemon))
@@ -408,6 +414,22 @@ func (v *vaultClient) buildClient() error {
 	return nil
 }
 
+// getVaultInitStatus is used to get the init status. It first clears the namespace header, to work around an
+// issue in Vault, then restores it.
+func (v *vaultClient) getVaultInitStatus() (bool, error) {
+	v.l.Lock()
+	defer v.l.Unlock()
+
+	// workaround for Vault behavior where namespace header causes /v1/sys/init (and other) endpoints to fail
+	if ns := v.client.Headers().Get(vaultconsts.NamespaceHeaderName); ns != "" {
+		v.client.SetNamespace("")
+		defer func() {
+			v.client.SetNamespace(ns)
+		}()
+	}
+	return v.client.Sys().InitStatus()
+}
+
 // establishConnection is used to make first contact with Vault. This should be
 // called in a go-routine since the connection is retried until the Vault Client
 // is stopped or the connection is successfully made at which point the renew
@@ -425,7 +447,7 @@ OUTER:
 		case <-retryTimer.C:
 			// Ensure the API is reachable
 			if !initStatus {
-				if _, err := v.client.Sys().InitStatus(); err != nil {
+				if _, err := v.getVaultInitStatus(); err != nil {
 					v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err)
 					retryTimer.Reset(v.config.ConnectionRetryIntv)
 					continue OUTER

nomad/vault_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/testutil"
 	vapi "github.com/hashicorp/vault/api"
+	vaultconsts "github.com/hashicorp/vault/helper/consts"
 )
 
 const (
@@ -175,6 +176,30 @@ func TestVaultClient_BadConfig(t *testing.T) {
 	}
 }
 
+// TestVaultClient_NamespaceSupport tests that the Vault namespace config, if present, will result in the
+// namespace header being set on the created Vault client.
+func TestVaultClient_NamespaceSupport(t *testing.T) {
+	t.Parallel()
+	require := require.New(t)
+	tr := true
+	testNs := "test-namespace"
+	conf := &config.VaultConfig{
+		Addr:      "https://vault.service.consul:8200",
+		Enabled:   &tr,
+		Token:     "testvaulttoken",
+		Namespace: testNs,
+	}
+	logger := testlog.HCLogger(t)
+
+	// Should be no error since Vault is not enabled
+	c, err := NewVaultClient(conf, logger, nil)
+	if err != nil {
+		t.Fatalf("failed to build vault client: %v", err)
+	}
+
+	require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName))
+}
+
 // started separately.
 // Test that the Vault Client can establish a connection even if it is started
 // before Vault is available.