From 04acc0c3caecd362aa3584c0660c3712acd9e49b Mon Sep 17 00:00:00 2001 From: Michael Gmelin Date: Sat, 22 Jan 2022 14:57:07 +0100 Subject: [PATCH 1/2] Add file_perms parameter to job's vault stanza At the moment, `/secrets/vault_token` is written using the nomad process' umask, which - by default - is `0022`, which results in writing the vault_token world-readable (`0644` / `rw-r--r--`). This patch aims to address this by allowing to change the permissions used from the default `666` (before umask). The name `file_perms` was chosen to distinguish this from the parameter `perms` in the `template` stanza (as templates always operate on files, while Vault tokens are about permissions as well, which would've been confusing otherwise). Example of use: vault { policies = ["nomad-tls-policy"] change_mode = "noop" file_perms = "600" } I tried to extend unit tests to include basic checks for the feature, not sure if this is sufficient, though. Resolves hashicorp/nomad#11900 --- api/tasks.go | 4 +++ client/allocrunner/taskrunner/vault_hook.go | 17 ++++++++++- command/agent/job_endpoint.go | 1 + command/agent/job_endpoint_test.go | 2 ++ jobspec/parse.go | 1 + jobspec/parse_test.go | 1 + jobspec/test-fixtures/basic.hcl | 1 + nomad/structs/diff_test.go | 30 +++++++++++++++++++ nomad/structs/structs.go | 11 +++++++ .../content/docs/job-specification/vault.mdx | 17 ++++++++--- 10 files changed, 80 insertions(+), 5 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index 10629aa2d2b..dec5f34f002 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -856,6 +856,7 @@ type Vault struct { Env *bool `hcl:"env,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` + FilePerms *string `mapstructure:"file_perms" hcl:"file_perms,optional"` } func (v *Vault) Canonicalize() { @@ -871,6 +872,9 @@ func (v *Vault) Canonicalize() { if v.ChangeSignal == nil { v.ChangeSignal = stringToPtr("SIGHUP") } + if v.FilePerms == nil { + v.FilePerms = stringToPtr("0666") + } } // NewTask creates and initializes a new Task. diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 8aa33a429dc..cf5d9c404d6 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -3,9 +3,11 @@ package taskrunner import ( "context" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" + "strconv" "sync" "time" @@ -81,6 +83,9 @@ type vaultHook struct { // tokenPath is the path in which to read and write the token tokenPath string + // tokenFilePerms are the file permissions applied to tokenPath + tokenPerms fs.FileMode + // alloc is the allocation alloc *structs.Allocation @@ -103,6 +108,7 @@ func newVaultHook(config *vaultHookConfig) *vaultHook { lifecycle: config.lifecycle, updater: config.updater, alloc: config.alloc, + tokenPerms: 0666, taskName: config.task, firstRun: true, ctx: ctx, @@ -126,6 +132,15 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe return nil } + // Set vaultTokenPath file permissions + if h.vaultStanza.FilePerms != "" { + v, err := strconv.ParseUint(h.vaultStanza.FilePerms, 8, 12) + if err != nil { + return fmt.Errorf("Failed to parse %q as octal: %v", h.vaultStanza.FilePerms, err) + } + h.tokenPerms = fs.FileMode(v) + } + // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" @@ -343,7 +358,7 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0666); err != nil { + if err := ioutil.WriteFile(h.tokenPath, []byte(token), h.tokenPerms); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b3a4b6bd225..ac07e220d24 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1140,6 +1140,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Env: *apiTask.Vault.Env, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, + FilePerms: *apiTask.Vault.FilePerms, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index d0c8a869095..34bd9ab7303 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2485,6 +2485,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: helper.BoolToPtr(true), ChangeMode: helper.StringToPtr("c"), ChangeSignal: helper.StringToPtr("sighup"), + FilePerms: helper.StringToPtr("0666"), }, Templates: []*api.Template{ { @@ -2882,6 +2883,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: true, ChangeMode: "c", ChangeSignal: "sighup", + FilePerms: "0666", }, Templates: []*structs.Template{ { diff --git a/jobspec/parse.go b/jobspec/parse.go index aad9d9fbe1b..f7ec01f574a 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -510,6 +510,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "env", "change_mode", "change_signal", + "file_perms", } if err := checkHCLKeys(listVal, valid); err != nil { return multierror.Prefix(err, "vault ->") diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 63cac85dfd7..e9e362e1eac 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -402,6 +402,7 @@ func TestParse(t *testing.T) { Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), + FilePerms: stringToPtr("644"), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 325c5e3624a..521738ef573 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -349,6 +349,7 @@ job "binstore-storagelocker" { env = false change_mode = "signal" change_signal = "SIGUSR1" + file_perms = "644" } } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index ef950b90d42..55eddd6455d 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6624,6 +6624,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + FilePerms: "0644", }, }, Expected: &TaskDiff{ @@ -6651,6 +6652,12 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "true", }, + { + Type: DiffTypeAdded, + Name: "FilePerms", + Old: "", + New: "0644", + }, }, Objects: []*ObjectDiff{ { @@ -6684,6 +6691,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + FilePerms: "0644", }, }, New: &Task{}, @@ -6712,6 +6720,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "", }, + { + Type: DiffTypeDeleted, + Name: "FilePerms", + Old: "0644", + New: "", + }, }, Objects: []*ObjectDiff{ { @@ -6746,6 +6760,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + FilePerms: "0666", }, }, New: &Task{ @@ -6755,6 +6770,7 @@ func TestTaskDiff(t *testing.T) { Env: false, ChangeMode: "restart", ChangeSignal: "foo", + FilePerms: "0644", }, }, Expected: &TaskDiff{ @@ -6782,6 +6798,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "false", }, + { + Type: DiffTypeEdited, + Name: "FilePerms", + Old: "0666", + New: "0644", + }, { Type: DiffTypeEdited, Name: "Namespace", @@ -6823,6 +6845,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + FilePerms: "", }, }, New: &Task{ @@ -6832,6 +6855,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + FilePerms: "", }, }, Expected: &TaskDiff{ @@ -6859,6 +6883,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "true", }, + { + Type: DiffTypeNone, + Name: "FilePerms", + Old: "", + New: "", + }, { Type: DiffTypeNone, Name: "Namespace", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 2c20932a24f..76c599bd1bf 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8851,12 +8851,16 @@ type Vault struct { // ChangeSignal is the signal sent to the task when a new token is // retrieved. This is only valid when using the signal change mode. ChangeSignal string + + // FilePerms is the file permissions vault_token should be written out with. + FilePerms string } func DefaultVaultBlock() *Vault { return &Vault{ Env: true, ChangeMode: VaultChangeModeRestart, + FilePerms: "0666", } } @@ -8904,6 +8908,13 @@ func (v *Vault) Validate() error { _ = multierror.Append(&mErr, fmt.Errorf("Unknown change mode %q", v.ChangeMode)) } + // Verify vault_token file permissions + if v.FilePerms != "" { + if _, err := strconv.ParseUint(v.FilePerms, 8, 12); err != nil { + _ = multierror.Append(&mErr, fmt.Errorf("Failed to parse %q as octal: %v", v.FilePerms, err)) + } + } + return mErr.ErrorOrNil() } diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index e6ab5ac6a86..971fcacb23a 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -34,6 +34,7 @@ job "docs" { change_mode = "signal" change_signal = "SIGUSR1" + file_perms = "600" } } } @@ -41,10 +42,12 @@ job "docs" { ``` The Nomad client will make the Vault token available to the task by writing it -to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN` -environment variable. If the Nomad cluster is [configured](/docs/configuration/vault#namespace) -to use [Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), -a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set. +to the secret directory at `secrets/vault_token` with access permissions as +specified in `file_perms` and by injecting a `VAULT_TOKEN` environment variable. If the +Nomad cluster is [configured](/docs/configuration/vault#namespace) to use +[Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), +a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` +is set. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -78,6 +81,12 @@ with Vault as well. the task requires. The Nomad client will retrieve a Vault token that is limited to those policies. +- `file_perms` `(string: "666")` - Specifies file permissions when creating + `secrets/vault_token`, the file containing the Vault token retrieved by + Nomad. File permissions are given in Unix octal notation _before applying + the Nomad process' umask_. Given the common _umask_ of `0022`, default + `file_perms` result in effective file permissions of `644` (`rw-r--r--`). + ## `vault` Examples The following examples only show the `vault` stanzas. Remember that the From 5e52838d3753b27f3a211f35f9665b4e8986796e Mon Sep 17 00:00:00 2001 From: Michael Gmelin Date: Tue, 25 Jan 2022 12:18:26 +0100 Subject: [PATCH 2/2] Add new `file` parameter to `vault` stanza This parameter controls whether the Vault token is actually written to `secrets/vault_token`. The Vault token is always written to `/private/vault_token`, `private` being a new directory to hold private secrets data not to be shared with the chroot and also not to be shared with Nomad UI. If `file` is set to `true` (the default), it is also written to `/secrets/vault_token`, using the permissions configured in `file_perms`. There are two rationales for this design: 1. Have one authoritative place where `vault_token` exists. 2. Don't read `vault_token` from a place that operates on a lower security level on Nomad restart. Ultimately, `secrets/vault_token` could have been altered in unfortunate or malicious ways from within the chroot. This was the primary reason to add an additional write operation instead of just modifying `tokenPath` depending on the value of `file`. Open questions: - Is creating a separate `private` directory worth the overhead? --- api/tasks.go | 4 ++ client/allocdir/alloc_dir.go | 15 +++++++ client/allocdir/task_dir.go | 14 +++++++ .../taskrunner/task_runner_test.go | 4 +- client/allocrunner/taskrunner/vault_hook.go | 42 ++++++++++++------- command/agent/job_endpoint.go | 1 + command/agent/job_endpoint_test.go | 2 + .../shared/executor/executor_linux_test.go | 1 + jobspec/parse.go | 1 + jobspec/parse_group.go | 2 + jobspec/parse_job.go | 2 + jobspec/parse_task.go | 2 + jobspec/parse_test.go | 9 ++++ jobspec/test-fixtures/basic.hcl | 1 + jobspec/test-fixtures/vault_inheritance.hcl | 1 + jobspec2/parse_job.go | 6 +++ nomad/structs/diff_test.go | 30 +++++++++++++ nomad/structs/structs.go | 12 ++---- .../content/docs/job-specification/vault.mdx | 5 ++- 19 files changed, 127 insertions(+), 27 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index dec5f34f002..2a6d9877124 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -856,6 +856,7 @@ type Vault struct { Env *bool `hcl:"env,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` + File *bool `mapstructure:"file" hcl:"file,optional"` FilePerms *string `mapstructure:"file_perms" hcl:"file_perms,optional"` } @@ -872,6 +873,9 @@ func (v *Vault) Canonicalize() { if v.ChangeSignal == nil { v.ChangeSignal = stringToPtr("SIGHUP") } + if v.File == nil { + v.File = boolToPtr(true) + } if v.FilePerms == nil { v.FilePerms = stringToPtr("0666") } diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 0dc52029cad..612346538d1 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -58,6 +58,10 @@ var ( // directory TaskSecrets = "secrets" + // TaskPrivate is the name of the pruvate directory inside each task + // directory + TaskPrivate = "private" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777} @@ -304,6 +308,13 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.PrivateDir) { + if err := removeSecretDir(dir.PrivateDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err)) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) @@ -441,6 +452,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { d.mu.RUnlock() return nil, fmt.Errorf("Reading secret file prohibited: %s", path) } + if filepath.HasPrefix(p, dir.PrivateDir) { + d.mu.RUnlock() + return nil, fmt.Errorf("Reading private file prohibited: %s", path) + } } d.mu.RUnlock() diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index d516c313cf1..7b9b081e9e5 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -39,6 +39,10 @@ type TaskDir struct { // /secrets/ SecretsDir string + // PrivateDir is the path to private/ directory on the host + // /private/ + PrivateDir string + // skip embedding these paths in chroots. Used for avoiding embedding // client.alloc_dir recursively. skip map[string]struct{} @@ -66,6 +70,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), + PrivateDir: filepath.Join(taskDir, TaskPrivate), skip: skip, logger: logger, } @@ -128,6 +133,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { return err } + // Create the private directory + if err := createSecretDir(t.PrivateDir); err != nil { + return err + } + + if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil { + return err + } + // Build chroot if chroot filesystem isolation is going to be used if createChroot { if err := t.buildChroot(chroot); err != nil { diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 9b60458b4b0..c949a6169d5 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1557,7 +1557,7 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { require.False(t, finalState.Failed) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) @@ -1622,7 +1622,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { require.Equal(t, 1, count) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index cf5d9c404d6..47203f330d4 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -83,8 +83,12 @@ type vaultHook struct { // tokenPath is the path in which to read and write the token tokenPath string + // sharedTokenPath is the path in which to only write, but never + // read the token from + sharedTokenPath string + // tokenFilePerms are the file permissions applied to tokenPath - tokenPerms fs.FileMode + sharedTokenPerms fs.FileMode // alloc is the allocation alloc *structs.Allocation @@ -102,18 +106,18 @@ type vaultHook struct { func newVaultHook(config *vaultHookConfig) *vaultHook { ctx, cancel := context.WithCancel(context.Background()) h := &vaultHook{ - vaultStanza: config.vaultStanza, - client: config.client, - eventEmitter: config.events, - lifecycle: config.lifecycle, - updater: config.updater, - alloc: config.alloc, - tokenPerms: 0666, - taskName: config.task, - firstRun: true, - ctx: ctx, - cancel: cancel, - future: newTokenFuture(), + vaultStanza: config.vaultStanza, + client: config.client, + eventEmitter: config.events, + lifecycle: config.lifecycle, + updater: config.updater, + alloc: config.alloc, + sharedTokenPerms: 0666, + taskName: config.task, + firstRun: true, + ctx: ctx, + cancel: cancel, + future: newTokenFuture(), } h.logger = config.logger.Named(h.Name()) return h @@ -138,13 +142,13 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe if err != nil { return fmt.Errorf("Failed to parse %q as octal: %v", h.vaultStanza.FilePerms, err) } - h.tokenPerms = fs.FileMode(v) + h.sharedTokenPerms = fs.FileMode(v) } // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" - h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) + h.tokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(h.tokenPath) if err != nil { if !os.IsNotExist(err) { @@ -156,6 +160,7 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Store the recovered token recoveredToken = string(data) } + h.sharedTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) // Launch the token manager go h.run(recoveredToken) @@ -358,9 +363,14 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := ioutil.WriteFile(h.tokenPath, []byte(token), h.tokenPerms); err != nil { + if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0600); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } + if h.vaultStanza.File { + if err := ioutil.WriteFile(h.sharedTokenPath, []byte(token), h.sharedTokenPerms); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + } return nil } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index ac07e220d24..dae5c3d7ba4 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1140,6 +1140,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Env: *apiTask.Vault.Env, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, + File: *apiTask.Vault.File, FilePerms: *apiTask.Vault.FilePerms, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 34bd9ab7303..f4e80960d31 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2485,6 +2485,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: helper.BoolToPtr(true), ChangeMode: helper.StringToPtr("c"), ChangeSignal: helper.StringToPtr("sighup"), + File: helper.BoolToPtr(true), FilePerms: helper.StringToPtr("0666"), }, Templates: []*api.Template{ @@ -2883,6 +2884,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: true, ChangeMode: "c", ChangeSignal: "sighup", + File: true, FilePerms: "0666", }, Templates: []*structs.Template{ diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 687c646350e..f003ae846d3 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -229,6 +229,7 @@ etc/ lib/ lib64/ local/ +private/ proc/ secrets/ sys/ diff --git a/jobspec/parse.go b/jobspec/parse.go index f7ec01f574a..b960e163cd3 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -510,6 +510,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "env", "change_mode", "change_signal", + "file", "file_perms", } if err := checkHCLKeys(listVal, valid); err != nil { diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 060af851f55..46a9083e564 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -211,6 +211,8 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { tgVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(tgVault, o); err != nil { diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go index b59b3b59456..bbe2ba2cf48 100644 --- a/jobspec/parse_job.go +++ b/jobspec/parse_job.go @@ -193,6 +193,8 @@ func parseJob(result *api.Job, list *ast.ObjectList) error { jobVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(jobVault, o); err != nil { diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index ff81b6ba66a..6545afc498f 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -291,6 +291,8 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) { v := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(v, o); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index e9e362e1eac..521c5560dc2 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -350,6 +350,8 @@ func TestParse(t *testing.T) { Policies: []string{"foo", "bar"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, Templates: []*api.Template{ { @@ -402,6 +404,7 @@ func TestParse(t *testing.T) { Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), + File: boolToPtr(false), FilePerms: stringToPtr("644"), }, }, @@ -762,6 +765,8 @@ func TestParse(t *testing.T) { Policies: []string{"group"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, }, { @@ -770,6 +775,8 @@ func TestParse(t *testing.T) { Policies: []string{"task"}, Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(false), + FilePerms: stringToPtr("0666"), }, }, }, @@ -783,6 +790,8 @@ func TestParse(t *testing.T) { Policies: []string{"job"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 521738ef573..81c99b69070 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -349,6 +349,7 @@ job "binstore-storagelocker" { env = false change_mode = "signal" change_signal = "SIGUSR1" + file = false file_perms = "644" } } diff --git a/jobspec/test-fixtures/vault_inheritance.hcl b/jobspec/test-fixtures/vault_inheritance.hcl index 18d83d9f5de..7425cea21de 100644 --- a/jobspec/test-fixtures/vault_inheritance.hcl +++ b/jobspec/test-fixtures/vault_inheritance.hcl @@ -14,6 +14,7 @@ job "example" { vault { policies = ["task"] env = false + file = false } } } diff --git a/jobspec2/parse_job.go b/jobspec2/parse_job.go index 9b533874f50..fda68885f1e 100644 --- a/jobspec2/parse_job.go +++ b/jobspec2/parse_job.go @@ -64,6 +64,12 @@ func normalizeVault(v *api.Vault) { if v.ChangeMode == nil { v.ChangeMode = stringToPtr("restart") } + if v.File == nil { + v.File = boolToPtr(true) + } + if v.FilePerms == nil { + v.FilePerms = stringToPtr("0666") + } } func normalizeNetworkPorts(networks []*api.NetworkResource) { diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 55eddd6455d..caf2aba05af 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6624,6 +6624,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, FilePerms: "0644", }, }, @@ -6652,6 +6653,12 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "true", }, + { + Type: DiffTypeAdded, + Name: "File", + Old: "", + New: "true", + }, { Type: DiffTypeAdded, Name: "FilePerms", @@ -6691,6 +6698,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, FilePerms: "0644", }, }, @@ -6720,6 +6728,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "", }, + { + Type: DiffTypeDeleted, + Name: "File", + Old: "true", + New: "", + }, { Type: DiffTypeDeleted, Name: "FilePerms", @@ -6760,6 +6774,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, FilePerms: "0666", }, }, @@ -6770,6 +6785,7 @@ func TestTaskDiff(t *testing.T) { Env: false, ChangeMode: "restart", ChangeSignal: "foo", + File: false, FilePerms: "0644", }, }, @@ -6798,6 +6814,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "false", }, + { + Type: DiffTypeEdited, + Name: "File", + Old: "true", + New: "false", + }, { Type: DiffTypeEdited, Name: "FilePerms", @@ -6845,6 +6867,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, FilePerms: "", }, }, @@ -6855,6 +6878,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, FilePerms: "", }, }, @@ -6883,6 +6907,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "true", }, + { + Type: DiffTypeNone, + Name: "File", + Old: "true", + New: "true", + }, { Type: DiffTypeNone, Name: "FilePerms", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 76c599bd1bf..05d37e5b347 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8852,18 +8852,14 @@ type Vault struct { // retrieved. This is only valid when using the signal change mode. ChangeSignal string + // File marks whether the Vault Token should be exposed in the file + // vault_token in the task's secrets directory. + File bool + // FilePerms is the file permissions vault_token should be written out with. FilePerms string } -func DefaultVaultBlock() *Vault { - return &Vault{ - Env: true, - ChangeMode: VaultChangeModeRestart, - FilePerms: "0666", - } -} - // Copy returns a copy of this Vault block. func (v *Vault) Copy() *Vault { if v == nil { diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index 971fcacb23a..691052df6fe 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -47,7 +47,7 @@ specified in `file_perms` and by injecting a `VAULT_TOKEN` environment variable. Nomad cluster is [configured](/docs/configuration/vault#namespace) to use [Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` -is set. +is set. This behavior can be altered using the `env` and `file` parameters. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -81,6 +81,9 @@ with Vault as well. the task requires. The Nomad client will retrieve a Vault token that is limited to those policies. +- `file` `(bool: true)` - Specifies if the Vault token should be written to + `secrets/vault_token`. + - `file_perms` `(string: "666")` - Specifies file permissions when creating `secrets/vault_token`, the file containing the Vault token retrieved by Nomad. File permissions are given in Unix octal notation _before applying