diff --git a/api/tasks.go b/api/tasks.go index 10629aa2d2b..2a6d9877124 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -856,6 +856,8 @@ type Vault struct { Env *bool `hcl:"env,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` + File *bool `mapstructure:"file" hcl:"file,optional"` + FilePerms *string `mapstructure:"file_perms" hcl:"file_perms,optional"` } func (v *Vault) Canonicalize() { @@ -871,6 +873,12 @@ func (v *Vault) Canonicalize() { if v.ChangeSignal == nil { v.ChangeSignal = stringToPtr("SIGHUP") } + if v.File == nil { + v.File = boolToPtr(true) + } + if v.FilePerms == nil { + v.FilePerms = stringToPtr("0666") + } } // NewTask creates and initializes a new Task. diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 0dc52029cad..612346538d1 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -58,6 +58,10 @@ var ( // directory TaskSecrets = "secrets" + // TaskPrivate is the name of the pruvate directory inside each task + // directory + TaskPrivate = "private" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777} @@ -304,6 +308,13 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.PrivateDir) { + if err := removeSecretDir(dir.PrivateDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err)) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) @@ -441,6 +452,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { d.mu.RUnlock() return nil, fmt.Errorf("Reading secret file prohibited: %s", path) } + if filepath.HasPrefix(p, dir.PrivateDir) { + d.mu.RUnlock() + return nil, fmt.Errorf("Reading private file prohibited: %s", path) + } } d.mu.RUnlock() diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index d516c313cf1..7b9b081e9e5 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -39,6 +39,10 @@ type TaskDir struct { // /secrets/ SecretsDir string + // PrivateDir is the path to private/ directory on the host + // /private/ + PrivateDir string + // skip embedding these paths in chroots. Used for avoiding embedding // client.alloc_dir recursively. skip map[string]struct{} @@ -66,6 +70,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), + PrivateDir: filepath.Join(taskDir, TaskPrivate), skip: skip, logger: logger, } @@ -128,6 +133,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { return err } + // Create the private directory + if err := createSecretDir(t.PrivateDir); err != nil { + return err + } + + if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil { + return err + } + // Build chroot if chroot filesystem isolation is going to be used if createChroot { if err := t.buildChroot(chroot); err != nil { diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 9b60458b4b0..c949a6169d5 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1557,7 +1557,7 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { require.False(t, finalState.Failed) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) @@ -1622,7 +1622,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { require.Equal(t, 1, count) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 8aa33a429dc..47203f330d4 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -3,9 +3,11 @@ package taskrunner import ( "context" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" + "strconv" "sync" "time" @@ -81,6 +83,13 @@ type vaultHook struct { // tokenPath is the path in which to read and write the token tokenPath string + // sharedTokenPath is the path in which to only write, but never + // read the token from + sharedTokenPath string + + // tokenFilePerms are the file permissions applied to tokenPath + sharedTokenPerms fs.FileMode + // alloc is the allocation alloc *structs.Allocation @@ -97,17 +106,18 @@ type vaultHook struct { func newVaultHook(config *vaultHookConfig) *vaultHook { ctx, cancel := context.WithCancel(context.Background()) h := &vaultHook{ - vaultStanza: config.vaultStanza, - client: config.client, - eventEmitter: config.events, - lifecycle: config.lifecycle, - updater: config.updater, - alloc: config.alloc, - taskName: config.task, - firstRun: true, - ctx: ctx, - cancel: cancel, - future: newTokenFuture(), + vaultStanza: config.vaultStanza, + client: config.client, + eventEmitter: config.events, + lifecycle: config.lifecycle, + updater: config.updater, + alloc: config.alloc, + sharedTokenPerms: 0666, + taskName: config.task, + firstRun: true, + ctx: ctx, + cancel: cancel, + future: newTokenFuture(), } h.logger = config.logger.Named(h.Name()) return h @@ -126,10 +136,19 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe return nil } + // Set vaultTokenPath file permissions + if h.vaultStanza.FilePerms != "" { + v, err := strconv.ParseUint(h.vaultStanza.FilePerms, 8, 12) + if err != nil { + return fmt.Errorf("Failed to parse %q as octal: %v", h.vaultStanza.FilePerms, err) + } + h.sharedTokenPerms = fs.FileMode(v) + } + // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" - h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) + h.tokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(h.tokenPath) if err != nil { if !os.IsNotExist(err) { @@ -141,6 +160,7 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Store the recovered token recoveredToken = string(data) } + h.sharedTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) // Launch the token manager go h.run(recoveredToken) @@ -343,9 +363,14 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0666); err != nil { + if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0600); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } + if h.vaultStanza.File { + if err := ioutil.WriteFile(h.sharedTokenPath, []byte(token), h.sharedTokenPerms); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + } return nil } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b3a4b6bd225..dae5c3d7ba4 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1140,6 +1140,8 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Env: *apiTask.Vault.Env, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, + File: *apiTask.Vault.File, + FilePerms: *apiTask.Vault.FilePerms, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index d0c8a869095..f4e80960d31 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2485,6 +2485,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: helper.BoolToPtr(true), ChangeMode: helper.StringToPtr("c"), ChangeSignal: helper.StringToPtr("sighup"), + File: helper.BoolToPtr(true), + FilePerms: helper.StringToPtr("0666"), }, Templates: []*api.Template{ { @@ -2882,6 +2884,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: true, ChangeMode: "c", ChangeSignal: "sighup", + File: true, + FilePerms: "0666", }, Templates: []*structs.Template{ { diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 687c646350e..f003ae846d3 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -229,6 +229,7 @@ etc/ lib/ lib64/ local/ +private/ proc/ secrets/ sys/ diff --git a/jobspec/parse.go b/jobspec/parse.go index aad9d9fbe1b..b960e163cd3 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -510,6 +510,8 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "env", "change_mode", "change_signal", + "file", + "file_perms", } if err := checkHCLKeys(listVal, valid); err != nil { return multierror.Prefix(err, "vault ->") diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 060af851f55..46a9083e564 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -211,6 +211,8 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { tgVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(tgVault, o); err != nil { diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go index b59b3b59456..bbe2ba2cf48 100644 --- a/jobspec/parse_job.go +++ b/jobspec/parse_job.go @@ -193,6 +193,8 @@ func parseJob(result *api.Job, list *ast.ObjectList) error { jobVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(jobVault, o); err != nil { diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index ff81b6ba66a..6545afc498f 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -291,6 +291,8 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) { v := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), } if err := parseVault(v, o); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 63cac85dfd7..521c5560dc2 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -350,6 +350,8 @@ func TestParse(t *testing.T) { Policies: []string{"foo", "bar"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, Templates: []*api.Template{ { @@ -402,6 +404,8 @@ func TestParse(t *testing.T) { Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), + File: boolToPtr(false), + FilePerms: stringToPtr("644"), }, }, }, @@ -761,6 +765,8 @@ func TestParse(t *testing.T) { Policies: []string{"group"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, }, { @@ -769,6 +775,8 @@ func TestParse(t *testing.T) { Policies: []string{"task"}, Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(false), + FilePerms: stringToPtr("0666"), }, }, }, @@ -782,6 +790,8 @@ func TestParse(t *testing.T) { Policies: []string{"job"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), + FilePerms: stringToPtr("0666"), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 325c5e3624a..81c99b69070 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -349,6 +349,8 @@ job "binstore-storagelocker" { env = false change_mode = "signal" change_signal = "SIGUSR1" + file = false + file_perms = "644" } } diff --git a/jobspec/test-fixtures/vault_inheritance.hcl b/jobspec/test-fixtures/vault_inheritance.hcl index 18d83d9f5de..7425cea21de 100644 --- a/jobspec/test-fixtures/vault_inheritance.hcl +++ b/jobspec/test-fixtures/vault_inheritance.hcl @@ -14,6 +14,7 @@ job "example" { vault { policies = ["task"] env = false + file = false } } } diff --git a/jobspec2/parse_job.go b/jobspec2/parse_job.go index 9b533874f50..fda68885f1e 100644 --- a/jobspec2/parse_job.go +++ b/jobspec2/parse_job.go @@ -64,6 +64,12 @@ func normalizeVault(v *api.Vault) { if v.ChangeMode == nil { v.ChangeMode = stringToPtr("restart") } + if v.File == nil { + v.File = boolToPtr(true) + } + if v.FilePerms == nil { + v.FilePerms = stringToPtr("0666") + } } func normalizeNetworkPorts(networks []*api.NetworkResource) { diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index ef950b90d42..caf2aba05af 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6624,6 +6624,8 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, + FilePerms: "0644", }, }, Expected: &TaskDiff{ @@ -6651,6 +6653,18 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "true", }, + { + Type: DiffTypeAdded, + Name: "File", + Old: "", + New: "true", + }, + { + Type: DiffTypeAdded, + Name: "FilePerms", + Old: "", + New: "0644", + }, }, Objects: []*ObjectDiff{ { @@ -6684,6 +6698,8 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, + FilePerms: "0644", }, }, New: &Task{}, @@ -6712,6 +6728,18 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "", }, + { + Type: DiffTypeDeleted, + Name: "File", + Old: "true", + New: "", + }, + { + Type: DiffTypeDeleted, + Name: "FilePerms", + Old: "0644", + New: "", + }, }, Objects: []*ObjectDiff{ { @@ -6746,6 +6774,8 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, + FilePerms: "0666", }, }, New: &Task{ @@ -6755,6 +6785,8 @@ func TestTaskDiff(t *testing.T) { Env: false, ChangeMode: "restart", ChangeSignal: "foo", + File: false, + FilePerms: "0644", }, }, Expected: &TaskDiff{ @@ -6782,6 +6814,18 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "false", }, + { + Type: DiffTypeEdited, + Name: "File", + Old: "true", + New: "false", + }, + { + Type: DiffTypeEdited, + Name: "FilePerms", + Old: "0666", + New: "0644", + }, { Type: DiffTypeEdited, Name: "Namespace", @@ -6823,6 +6867,8 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, + FilePerms: "", }, }, New: &Task{ @@ -6832,6 +6878,8 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, + FilePerms: "", }, }, Expected: &TaskDiff{ @@ -6859,6 +6907,18 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "true", }, + { + Type: DiffTypeNone, + Name: "File", + Old: "true", + New: "true", + }, + { + Type: DiffTypeNone, + Name: "FilePerms", + Old: "", + New: "", + }, { Type: DiffTypeNone, Name: "Namespace", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 2c20932a24f..05d37e5b347 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8851,13 +8851,13 @@ type Vault struct { // ChangeSignal is the signal sent to the task when a new token is // retrieved. This is only valid when using the signal change mode. ChangeSignal string -} -func DefaultVaultBlock() *Vault { - return &Vault{ - Env: true, - ChangeMode: VaultChangeModeRestart, - } + // File marks whether the Vault Token should be exposed in the file + // vault_token in the task's secrets directory. + File bool + + // FilePerms is the file permissions vault_token should be written out with. + FilePerms string } // Copy returns a copy of this Vault block. @@ -8904,6 +8904,13 @@ func (v *Vault) Validate() error { _ = multierror.Append(&mErr, fmt.Errorf("Unknown change mode %q", v.ChangeMode)) } + // Verify vault_token file permissions + if v.FilePerms != "" { + if _, err := strconv.ParseUint(v.FilePerms, 8, 12); err != nil { + _ = multierror.Append(&mErr, fmt.Errorf("Failed to parse %q as octal: %v", v.FilePerms, err)) + } + } + return mErr.ErrorOrNil() } diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index e6ab5ac6a86..691052df6fe 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -34,6 +34,7 @@ job "docs" { change_mode = "signal" change_signal = "SIGUSR1" + file_perms = "600" } } } @@ -41,10 +42,12 @@ job "docs" { ``` The Nomad client will make the Vault token available to the task by writing it -to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN` -environment variable. If the Nomad cluster is [configured](/docs/configuration/vault#namespace) -to use [Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), -a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set. +to the secret directory at `secrets/vault_token` with access permissions as +specified in `file_perms` and by injecting a `VAULT_TOKEN` environment variable. If the +Nomad cluster is [configured](/docs/configuration/vault#namespace) to use +[Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), +a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` +is set. This behavior can be altered using the `env` and `file` parameters. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -78,6 +81,15 @@ with Vault as well. the task requires. The Nomad client will retrieve a Vault token that is limited to those policies. +- `file` `(bool: true)` - Specifies if the Vault token should be written to + `secrets/vault_token`. + +- `file_perms` `(string: "666")` - Specifies file permissions when creating + `secrets/vault_token`, the file containing the Vault token retrieved by + Nomad. File permissions are given in Unix octal notation _before applying + the Nomad process' umask_. Given the common _umask_ of `0022`, default + `file_perms` result in effective file permissions of `644` (`rw-r--r--`). + ## `vault` Examples The following examples only show the `vault` stanzas. Remember that the