From 39439c5f12946d4e56e2e3583591c3f620ec0653 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Tue, 2 Apr 2019 19:49:20 +0000 Subject: [PATCH 01/13] config/docs: added `namespace` to vault config server/client: process `namespace` config, setting on the instantiated vault client --- client/vaultclient/vaultclient.go | 5 ++++ client/vaultclient/vaultclient_test.go | 23 +++++++++++++++++ nomad/structs/config/vault.go | 4 +++ nomad/vault.go | 4 +++ nomad/vault_test.go | 25 +++++++++++++++++++ .../source/docs/configuration/vault.html.md | 4 +++ 6 files changed, 65 insertions(+) diff --git a/client/vaultclient/vaultclient.go b/client/vaultclient/vaultclient.go index 2adcec1052f..5512882e977 100644 --- a/client/vaultclient/vaultclient.go +++ b/client/vaultclient/vaultclient.go @@ -163,6 +163,11 @@ func NewVaultClient(config *config.VaultConfig, logger hclog.Logger, tokenDerive "User-Agent": []string{"hashicorp/nomad"}, }) + // SetHeaders above will replace all headers, make this call second + if config.Namespace != "" { + client.SetNamespace(config.Namespace) + } + c.client = client return c, nil diff --git a/client/vaultclient/vaultclient_test.go b/client/vaultclient/vaultclient_test.go index ab5a7474e5e..6dad3778e8e 100644 --- a/client/vaultclient/vaultclient_test.go +++ b/client/vaultclient/vaultclient_test.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/testutil" vaultapi "github.com/hashicorp/vault/api" + vaultconsts "github.com/hashicorp/vault/helper/consts" "github.com/stretchr/testify/require" ) @@ -92,6 +93,28 @@ func TestVaultClient_TokenRenewals(t *testing.T) { } } +// TestVaultClient_NamespaceSupport tests that the Vault namespace config, if present, will result in the +// namespace header being set on the created Vault client. +func TestVaultClient_NamespaceSupport(t *testing.T) { + t.Parallel() + require := require.New(t) + tr := true + testNs := "test-namespace" + + logger := testlog.HCLogger(t) + + conf := config.DefaultConfig() + conf.VaultConfig.Enabled = &tr + conf.VaultConfig.Token = "testvaulttoken" + conf.VaultConfig.Namespace = testNs + c, err := NewVaultClient(conf.VaultConfig, logger, nil) + if err != nil { + t.Fatalf("failed to build vault client: %v", err) + } + + require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) +} + func TestVaultClient_Heap(t *testing.T) { t.Parallel() tr := true diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index bfb515623f5..d3e2028c5d4 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -37,6 +37,10 @@ type VaultConfig struct { // role the token is from. Role string `mapstructure:"create_from_role"` + // Namespaces sets the Vault namespace used for all calls against the + // Vault API. If this is unset, then Nomad does not use Vault namespaces. + Namespace string `mapstructure:"namespace"` + // AllowUnauthenticated allows users to submit jobs requiring Vault tokens // without providing a Vault token proving they have access to these // policies. diff --git a/nomad/vault.go b/nomad/vault.go index dc4060a20e4..872245077bd 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -252,6 +252,10 @@ func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVault return nil, err } + if c.Namespace != "" { + v.client.SetNamespace(c.Namespace) + } + // Launch the required goroutines v.tomb.Go(wrapNilError(v.establishConnection)) v.tomb.Go(wrapNilError(v.revokeDaemon)) diff --git a/nomad/vault_test.go b/nomad/vault_test.go index 0e977c5408f..6edaed1fc72 100644 --- a/nomad/vault_test.go +++ b/nomad/vault_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/testutil" vapi "github.com/hashicorp/vault/api" + vaultconsts "github.com/hashicorp/vault/helper/consts" ) const ( @@ -175,6 +176,30 @@ func TestVaultClient_BadConfig(t *testing.T) { } } +// TestVaultClient_NamespaceSupport tests that the Vault namespace config, if present, will result in the +// namespace header being set on the created Vault client. +func TestVaultClient_NamespaceSupport(t *testing.T) { + t.Parallel() + require := require.New(t) + tr := true + testNs := "test-namespace" + conf := &config.VaultConfig{ + Addr: "https://vault.service.consul:8200", + Enabled: &tr, + Token: "testvaulttoken", + Namespace: testNs, + } + logger := testlog.HCLogger(t) + + // Should be no error since Vault is not enabled + c, err := NewVaultClient(conf, logger, nil) + if err != nil { + t.Fatalf("failed to build vault client: %v", err) + } + + require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) +} + // started separately. // Test that the Vault Client can establish a connection even if it is started // before Vault is available. diff --git a/website/source/docs/configuration/vault.html.md b/website/source/docs/configuration/vault.html.md index 9fcee0d818c..d34e29a3a61 100644 --- a/website/source/docs/configuration/vault.html.md +++ b/website/source/docs/configuration/vault.html.md @@ -73,6 +73,10 @@ vault { - `key_file` `(string: "")` - Specifies the path to the private key used for Vault communication. If this is set then you need to also set `cert_file`. + +- `namespace` `(string: "")` - Specifies the [Vault namespace](https://www.vaultproject.io/docs/enterprise/namespaces/index.html) + used by the Vault integration. If non-empty, this namespace will be used on + all Vault API calls. - `tls_server_name` `(string: "")` - Specifies an optional string used to set the SNI host when connecting to Vault via TLS. From 3e29d7c65217da5e97a4476473cf30e6345df210 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Wed, 3 Apr 2019 11:58:34 +0000 Subject: [PATCH 02/13] wip: added config parsing support, CLI flag, still need more testing, VAULT_ var, documentation --- command/agent/command.go | 1 + command/agent/config_parse.go | 1 + nomad/structs/config/vault.go | 3 +++ nomad/vault.go | 26 ++++++++++++++++++++++---- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index a1658eb5069..1ac133e5345 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -163,6 +163,7 @@ func (c *Command) readConfig() *Config { }), "vault-allow-unauthenticated", "") flags.StringVar(&cmdConfig.Vault.Token, "vault-token", "", "") flags.StringVar(&cmdConfig.Vault.Addr, "vault-address", "", "") + flags.StringVar(&cmdConfig.Vault.Namespace, "vault-namespace", "", "") flags.StringVar(&cmdConfig.Vault.Role, "vault-create-from-role", "", "") flags.StringVar(&cmdConfig.Vault.TLSCaFile, "vault-ca-file", "", "") flags.StringVar(&cmdConfig.Vault.TLSCaPath, "vault-ca-path", "", "") diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 6df9e83d437..6e64aac141b 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -892,6 +892,7 @@ func parseVaultConfig(result **config.VaultConfig, list *ast.ObjectList) error { "tls_server_name", "tls_skip_verify", "token", + "namespace", } if err := helper.CheckHCLKeys(listVal, valid); err != nil { diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index d3e2028c5d4..c568a70ecfb 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -110,6 +110,9 @@ func (a *VaultConfig) Merge(b *VaultConfig) *VaultConfig { if b.Token != "" { result.Token = b.Token } + if b.Namespace != "" { + result.Namespace = b.Namespace + } if b.Role != "" { result.Role = b.Role } diff --git a/nomad/vault.go b/nomad/vault.go index 872245077bd..f2915345b4d 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -10,12 +10,13 @@ import ( "sync/atomic" "time" - tomb "gopkg.in/tomb.v2" + "gopkg.in/tomb.v2" - metrics "github.com/armon/go-metrics" + "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" vapi "github.com/hashicorp/vault/api" + vaultconsts "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" @@ -253,6 +254,7 @@ func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVault } if c.Namespace != "" { + logger.Debug("Setting Vault namespace", "namespace", c.Namespace) v.client.SetNamespace(c.Namespace) } @@ -412,6 +414,22 @@ func (v *vaultClient) buildClient() error { return nil } +// getVaultInitStatus is used to get the init status. It first clears the namespace header, to work around an +// issue in Vault, then restores it. +func (v *vaultClient) getVaultInitStatus() (bool, error) { + v.l.Lock() + defer v.l.Unlock() + + // workaround for Vault behavior where namespace header causes /v1/sys/init (and other) endpoints to fail + if ns := v.client.Headers().Get(vaultconsts.NamespaceHeaderName); ns != "" { + v.client.SetNamespace("") + defer func() { + v.client.SetNamespace(ns) + }() + } + return v.client.Sys().InitStatus() +} + // establishConnection is used to make first contact with Vault. This should be // called in a go-routine since the connection is retried until the Vault Client // is stopped or the connection is successfully made at which point the renew @@ -429,7 +447,7 @@ OUTER: case <-retryTimer.C: // Ensure the API is reachable if !initStatus { - if _, err := v.client.Sys().InitStatus(); err != nil { + if _, err := v.getVaultInitStatus(); err != nil { v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err) retryTimer.Reset(v.config.ConnectionRetryIntv) continue OUTER From 92871e268a5a8101ff020f61e9ef97a706f08e2f Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Thu, 4 Apr 2019 17:51:40 +0000 Subject: [PATCH 03/13] vendor: updated consul-template and downstream consul-template -> v0.20.0 consul/api -> v1.2.1 vault/api -> v1.0.3 go-retryablehttp -> v0.5.2 circonus-gometrics: modified local source for compat with go-retryablehttp --- .../circonus-gometrics/CHANGELOG.md | 12 + .../circonus-gometrics/README.md | 4 +- .../circonus-gometrics/api/worksheet.go | 20 +- .../circonus-gometrics/submit.go | 3 +- .../hashicorp/consul-template/CHANGELOG.md | 991 ++++++++ .../hashicorp/consul-template/Gopkg.lock | 696 ++++++ .../hashicorp/consul-template/Gopkg.toml | 64 + .../hashicorp/consul-template/Makefile | 236 ++ .../hashicorp/consul-template/README.md | 2214 +++++++++++++++++ .../hashicorp/consul-template/cli.go | 16 + .../hashicorp/consul-template/config/vault.go | 50 +- .../dependency/catalog_node.go | 2 + .../dependency/catalog_service.go | 2 + .../consul-template/dependency/client_set.go | 6 + .../dependency/health_service.go | 2 + .../dependency/vault_agent_token.go | 121 + .../dependency/vault_common.go | 91 +- .../consul-template/dependency/vault_read.go | 45 +- .../consul-template/manager/runner.go | 32 +- .../consul-template/renderer/file_perms.go | 22 + .../renderer/file_perms_windows.go | 9 + .../{manager => renderer}/renderer.go | 9 +- .../consul-template/template/funcs.go | 5 +- .../consul-template/version/version.go | 2 +- .../consul-template/watch/watcher.go | 13 + vendor/github.com/hashicorp/consul/NOTICE.md | 3 + .../github.com/hashicorp/consul/api/agent.go | 166 +- vendor/github.com/hashicorp/consul/api/api.go | 58 +- .../hashicorp/consul/api/catalog.go | 15 +- .../hashicorp/consul/api/connect.go | 12 + .../hashicorp/consul/api/connect_ca.go | 165 ++ .../hashicorp/consul/api/connect_intention.go | 302 +++ .../github.com/hashicorp/consul/api/health.go | 19 +- .../github.com/hashicorp/consul/api/lock.go | 5 +- .../hashicorp/consul/api/prepared_query.go | 8 + .../hashicorp/consul/api/semaphore.go | 5 +- .../hashicorp/go-retryablehttp/client.go | 36 +- .../hashicorp/go-retryablehttp/go.mod | 3 + .../hashicorp/go-retryablehttp/go.sum | 2 + .../hashicorp/vault/api/auth_token.go | 1 + .../github.com/hashicorp/vault/api/client.go | 42 +- .../hashicorp/vault/api/output_string.go | 69 + .../github.com/hashicorp/vault/api/request.go | 4 +- .../hashicorp/vault/api/ssh_agent.go | 6 +- .../hashicorp/vault/api/sys_auth.go | 48 +- .../hashicorp/vault/api/sys_health.go | 2 + .../hashicorp/vault/api/sys_leader.go | 1 + .../hashicorp/vault/api/sys_mounts.go | 19 +- .../hashicorp/vault/api/sys_plugins.go | 137 +- .../hashicorp/vault/api/sys_policy.go | 14 +- .../hashicorp/vault/api/sys_seal.go | 17 + .../vault/helper/consts/plugin_types.go | 59 + .../vault/helper/consts/replication.go | 35 +- vendor/vendor.json | 34 +- 54 files changed, 5798 insertions(+), 156 deletions(-) create mode 100644 vendor/github.com/hashicorp/consul-template/CHANGELOG.md create mode 100644 vendor/github.com/hashicorp/consul-template/Gopkg.lock create mode 100644 vendor/github.com/hashicorp/consul-template/Gopkg.toml create mode 100644 vendor/github.com/hashicorp/consul-template/Makefile create mode 100644 vendor/github.com/hashicorp/consul-template/README.md create mode 100644 vendor/github.com/hashicorp/consul-template/dependency/vault_agent_token.go create mode 100644 vendor/github.com/hashicorp/consul-template/renderer/file_perms.go create mode 100644 vendor/github.com/hashicorp/consul-template/renderer/file_perms_windows.go rename vendor/github.com/hashicorp/consul-template/{manager => renderer}/renderer.go (95%) create mode 100644 vendor/github.com/hashicorp/consul/NOTICE.md create mode 100644 vendor/github.com/hashicorp/consul/api/connect.go create mode 100644 vendor/github.com/hashicorp/consul/api/connect_ca.go create mode 100644 vendor/github.com/hashicorp/consul/api/connect_intention.go create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/go.mod create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/go.sum create mode 100644 vendor/github.com/hashicorp/vault/api/output_string.go create mode 100644 vendor/github.com/hashicorp/vault/helper/consts/plugin_types.go diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md b/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md index 581157989fd..4a0ab2c0d80 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md +++ b/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md @@ -1,3 +1,15 @@ +# v2.2.4 + +* fix: worksheet.graphs is a required attribute. worksheet.smart_queries is an optional attribute. + +# v2.2.3 + +* upd: remove go.{mod,dep} as cgm being v2 causes more issues than it solves at this point. will re-add after `go mod` becomes more common and adding `v2` to all internal import statements won't cause additional issues. + +# v2.2.2 + +* upd: add go.mod and go.sum + # v2.2.1 * fix: if submission url host is 'api.circonus.com' do not use private CA in TLSConfig diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/README.md b/vendor/github.com/circonus-labs/circonus-gometrics/README.md index 323f97c02a1..36192030938 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/README.md +++ b/vendor/github.com/circonus-labs/circonus-gometrics/README.md @@ -231,6 +231,4 @@ func main() { ``` -Unless otherwise noted, the source files are distributed under the BSD-style license found in the LICENSE file. - -[![codecov](https://codecov.io/gh/maier/circonus-gometrics/branch/master/graph/badge.svg)](https://codecov.io/gh/maier/circonus-gometrics) +Unless otherwise noted, the source files are distributed under the BSD-style license found in the [LICENSE](LICENSE) file. diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/api/worksheet.go b/vendor/github.com/circonus-labs/circonus-gometrics/api/worksheet.go index 5c33642030a..d9d9675f950 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/api/worksheet.go +++ b/vendor/github.com/circonus-labs/circonus-gometrics/api/worksheet.go @@ -30,19 +30,21 @@ type WorksheetSmartQuery struct { // Worksheet defines a worksheet. See https://login.circonus.com/resources/api/calls/worksheet for more information. type Worksheet struct { - CID string `json:"_cid,omitempty"` // string - Description *string `json:"description"` // string or null - Favorite bool `json:"favorite"` // boolean - Graphs []WorksheetGraph `json:"graphs"` // [] len >= 0 - Notes *string `json:"notes"` // string or null - SmartQueries []WorksheetSmartQuery `json:"smart_queries"` // [] len >= 0 - Tags []string `json:"tags"` // [] len >= 0 - Title string `json:"title"` // string + CID string `json:"_cid,omitempty"` // string + Description *string `json:"description"` // string or null + Favorite bool `json:"favorite"` // boolean + Graphs []WorksheetGraph `json:"graphs"` // [] len >= 0 + Notes *string `json:"notes"` // string or null + SmartQueries []WorksheetSmartQuery `json:"smart_queries,omitempty"` // [] len >= 0 + Tags []string `json:"tags"` // [] len >= 0 + Title string `json:"title"` // string } // NewWorksheet returns a new Worksheet (with defaults, if applicable) func NewWorksheet() *Worksheet { - return &Worksheet{} + return &Worksheet{ + Graphs: []WorksheetGraph{}, // graphs is a required attribute and cannot be null + } } // FetchWorksheet retrieves worksheet with passed cid. diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/submit.go b/vendor/github.com/circonus-labs/circonus-gometrics/submit.go index 151f6c0a44b..7af5adb3366 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/submit.go +++ b/vendor/github.com/circonus-labs/circonus-gometrics/submit.go @@ -141,7 +141,8 @@ func (m *CirconusMetrics) trapCall(payload []byte) (int, error) { client.CheckRetry = retryPolicy attempts := -1 - client.RequestLogHook = func(logger *log.Logger, req *http.Request, retryNumber int) { + + client.RequestLogHook = func(logger retryablehttp.Logger, req *http.Request, retryNumber int) { attempts = retryNumber } diff --git a/vendor/github.com/hashicorp/consul-template/CHANGELOG.md b/vendor/github.com/hashicorp/consul-template/CHANGELOG.md new file mode 100644 index 00000000000..89d2c19f720 --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/CHANGELOG.md @@ -0,0 +1,991 @@ +## v0.20.0 (February 19, 2019) + +IMPROVEMENTS: + +* Support for Consul service metadata [GH-1113] +* Support for Vault's KV v2 secrets engine, including versioned secrets [GH-1180]. +* Support for Vault Enterprise's namespaces feature [GH-1181]. +* Support for a new config parameter, `vault_agent_token_file`, which supports loading the Vault token from the contents of a dynamically updated file. This is intended for use in environments like Kubernetes [GH-1185]. +* A template's destination file will now have its user and group permissions preserved on supported OSes (Linux/MacOS) [GH-1061]. + +BUG FIXES: + +* The indent function no longer panics on negative spaces variable [GH-1127] +* Fixed an issue that caused `exec` to not be called with multiple templates and `wait` configured [GH-1043] +* Fixed an issue where Consul Template did not wait for most of a non-renewable secret's lease before attempting to refresh the secret. [GH-1183] + +## v0.19.5 (June 12, 2018) + +BUG FIXES: + + * The de-duplication feature was incorrectly calculating the hash of dependency + values over an unstable encoding of the data. This meant that in most cases + the templates were being re-written to KV and on all watching template + instances every minimum update time (i.e. `wait { min = X }`). At best this + was a lot of wasted work, in some cases it caused 100% CPU usage when template + instance leadership was split. [GH-1099, GH-1095] + * Fixed an issue where we waited unnecessarily for a child process to exit [GH-1101] + +IMPROVEMENTS: + + * Initiating runner log level moved to DEBUG [GH-1088] + + +## v0.19.4 (October 30, 2017) + +BREAKING CHANGES: + + * The version of Consul Template is now taken into account when using + de-duplication mode. Without bundling the version, it's challenging to + upgrade existing clusters or run multiple versions of Consul Template on the + same cluster and template simultaneously. [GH-1025] + +BUG FIXES: + + * Remove references to unsupported `dump_signal` configuration + + * Update vendor libraries to support Consul 1.0.0 changes for better test + stability + + * Renew unwrapped Vault token (previously Consul Template) would try to renew + the wrapped token, which would not work. + + * Do not sort results when `~near` queries are used [GH-1027] + + * Handle integer overflow in exponential backoff calculations + [GH-1031, GH-1028] + + * Properly preserve existing file permissions [GH-1037] + +IMPROVEMENTS: + + * Compile with Go 1.9.2 + + * The Vault grace period in the config is now set to 15 seconds as the + default. This matches Vault's default configuration for consistency. + + * Add `indent` function for indenting blocks of text in templates + + * Allow additional colons in the template command on the CLI [GH-1026] + + * Add Vault Transit example for key exfiltration [GH-1014] + + * Add a new option for disabling recursive directory creation per template + [GH-1033] + + * Allow dots in node names [GH-977] + +## v0.19.3 (September 11, 2017) + +BUG FIXES: + + * Fix a bug that would cause once mode to not exit when the file pre-existed + on disk with the correct contents. [GH-1000] + +## v0.19.2 (September 1, 2017) + +BUG FIXES: + + * Fix a critical bug that would cause a hot loop for some TTL durations. + [GH-1004] + +## v0.19.1 (August 25, 2017) + +IMPROVEMENTS: + + * The runner's render event now includes the last-rendered template contents. + This is useful when embedding Consul Template as a library. [GH-974-975] + + * Use the new Golang API renewer [GH-978] + + * Compile and build with Go 1.9 + +BUG FIXES: + + * Add per-template option `error_on_missing_key`. This causes the template to + error when the user attempts to access a key in a map or field in a struct + that does not exist. Previous behavior was to print ``, which + might not be the desired behavior. This is opt-in behavior on a + per-template basis. There is no global option. A future version of + Consul Template will switch the default behavior to this safer format, but + that change will be clearly called out as a breaking change in the future. + Users should set `error_on_missing_key = false` in their configuration + files if they are relying on the current `` behavior. + [GH-973, GH-972] + * Ensure all templates are rendered before spawning commands [GH-991, GH-995] + +## v0.19.0 (June 29, 2017) + +BREAKING CHANGES: + + * All previous deprecation errors have been removed and associated configs or + CLI options are no longer valid. It is highly recommended that you run + v0.18.5 and resolve any deprecations before upgrading to this version! + +IMPROVEMENTS: + + * Add new configuration option `vault.grace`, which configures the grace + period between lease renewal and secret re-acquisition. When renewing a + secret, if the remaining lease is less than or equal to the configured + grace, Consul Template will request a new credential. This prevents Vault + from revoking the credential at expiration and Consul Template having a + stale credential. **If you set this to a value that is higher than your + default TTL or max TTL, Consul Template will always read a new secret!** + * Add a new option to `datacenters` to optionally ignore inaccessible + datacenters [GH-908]. + +BUG FIXES: + + * Use the logger as soon as its available for output [GH-947] + * Update Consul API library to fix a bug where custom CA configuration was + ignored [GH-965] + + +## v0.18.5 (May 25, 2017) + +BREAKING CHANGES: + + * Retry now has a sane maximum default. Previous versions of Consul Template + would retry indefinitely, potentially allowing the time between retries to + reach days, months, or years due to the exponential nature. Users wishing + to use the old behavior should set `max_backoff = 0` in their + configurations. [GH-940] + +IMPROVEMENTS: + + * Add support for `MaxBackoff` in Retry options [GH-938, GH-939] + * Compile with Go 1.8.3 + +## v0.18.4 (May 25, 2017) + +BUG FIXES: + + * Compile with go 1.8.2 for the security fix. The code is exactly the same as + v0.18.3. + +## v0.18.3 (May 9, 2017) + +IMPROVEMENTS: + + * Add support for local datacenter in node queries [GH-862, GH-927] + * Add support for service tags on health checks [Consul vendor update] + +BUG FIXES: + + * Seed the random generator for splay values + * Reset retries counter on successful contact [GH-931] + * Return a nil slice instead of an error for non-existent maps + [GH-906, GH-932] + * Do not return data in dedup mode if the template is unchanged + [GH-933 GH-698] + +NOTABLE: + + * Consul Template is now built with Go 1.8.1 + * Update internal library to Consul 0.8.2 - this should not affect any users + +## v0.18.2 (March 28, 2017) + +IMPROVEMENTS: + + * Add missing HTTP transport configuration options + * Add `modulo` function for performing modulo math + +BUG FIXES: + + * Default transport max idle connections based on `GOMAXPROCS` + * Read `VAULT_*` envvars before finalizing [GH-914, GH-916] + * Register `[]*KeyPair` as a gob [GH-893] + +## v0.18.1 (February 7, 2017) + +IMPROVEMENTS: + + * Add support for tagged addresses and metadata [GH-863] + * Add `.exe` extension to Windows binaries [GH-875] + * Add support for customizing the low-level transport details for Consul and + Vault [GH-880, GH-877] + * Read token from `~/.vault-token` if it exists [GH-878, GH-884] + +BUG FIXES: + + * Resolve an issue with filters on health service dependencies [GH-857] + * Restore ability to reload configurations from disk [GH-866] + * Move `env` back to a helper function [GH-882] + + This was causing a lot of issues for users, and it required many folks to + re-write their templates for the small benefit of people running in + de-duplicate mode who did not understand the trade-offs. The README is now + updated with the trade-offs of running in dedup mode and the expected `env` + behavior has been restored. + + * Do not loop indefinitely if the dedup manager is unable to acquire a lock + [GH-864] + + +## v0.18.0 (January 20, 2017) + +NEW FEATURES: + + * Add new template function `keyExists` for determining if a key is present. + See the breaking change notice before for more information about the + motivation for this change. + + * Add `scratch` for storing information across a template invocation. Scratch + is especially useful when saving a computed value to use it across a + template. Scratch values are not shared across multiple templates and are + not persisted between template invocations + + * Add support for controlling retry behavior for failed communications to + Consul or Vault. By default, Consul Template will now retry 5 times before + returning an error. The backoff timing and number of attempts can be tuned + using the CLI or a configuration file. + + * Add `executeTemplate` function for executing a defined template. + + * Add `base64Decode`, `base64Encode`, `base64URLDecode`, and `base64URLEncode` + functions for working with base64 encodings. + + * Add `containsAll`, `containsAny`, `containsNone`, and `containsNotAll` + functions for easy filtering of multiple tag selections. + +BREAKING CHANGES: + + * Consul Template now **blocks on `key` queries**. The previous behavior was + to always pass through, allowing users to use the existence of a key as + a source of control flow. This caused confusion among many users, so we + have restored the expected behavior of blocking on a `key` query, but have + added `keyExists` to check for the existence of a key. Note that the + `keyOrDefault` function remains unchanged and will not block if the value + is nil, as expected. + + * The `vault` template function has been removed. This has been deprecated + with a warning since v0.14.0. + + * A shell is no longer assumed for Template commands. Previous versions of + Consul Template assumed `/bin/sh` (`cmd` on Windows) as the parent + process for the template command. Due to user requests and a desire to + customize the shell, Consul Template no longer wraps the command in a + shell. For most commands, this change will be transparent. If you were + utilizing shell-specific functions like `&&`, `||`, or conditionals, you + will need to wrap you command in a shell, for example: + + ```shell + -template "in.tpl:out.tpl:/bin/bash -c 'echo a || b'" + ``` + + or + + ```hcl + template { + command = "/bin/bash -c 'echo a || b'" + } + ``` + + * The `env` function is now treated as a dependency instead of a helper. For + most users, there will be no impact. + + * This release is compiled with Golang v1.8. We do not expect this to cause + any issues, but it is worth calling out. + +DEPRECATIONS: + + * `.Tags.Contains` is deprecated. Templates should make use of the built-in + `in` and `contains` functions instead. For example: + + ```liquid + {{ if .Tags.Contains "foo" }} + ``` + + becomes: + + ```liquid + {{ if .Tags | contains "foo" }} + ``` + + or: + + ```liquid + {{ if "foo" | in .Tags }} + ``` + + * `key_or_default` has been renamed to `keyOrDefault` to better align with + Go's naming structure. The old method is aliased and will remain until a + future release. + + * Consul-specific CLI options are now prefixed with `-consul-`: + + * `-auth` is now `-consul-auth` + * `-ssl-(.*)` is now `-consul-ssl-$1` + * `-retry` is now `-consul-retry` and has been broken apart into more + specific CLI options. + + * Consul-specific configuration options are now nested under a stanza. For + example: + + ```hcl + auth { + username = "foo" + password = "bar" + } + ``` + + becomes: + + ```hcl + consul { + auth { + username = "foo" + password = "bar" + } + } + ``` + + This applies to the `auth`, `retry`, `ssl`, and `token` options. + +IMPROVEMENTS: + + * Add CLI support for all SSL configuration options for both Consul and Vault. + Vault options are identical to Consul but with `vault-` prefix. Includes + the addition of `ssl-ca-path` to be consistent with file-based configuration + options. + + * `ssl` `vault-ssl` (Enable) + * `ssl-verify` `vault-ssl-verify` + * `ssl-cert` `vault-ssl-cert` + * `ssl-key` `vault-ssl-key` + * `ssl-ca-cert` `vault-ssl-ca-cert` + * `ssl-ca-path` `vault-ssl-ca-path` + * `ssl-server-name` `vault-ssl-server-name` + + * Add `-consul-ssl-server-name` + * Add `-consul-ssl-ca-path` + * Add `-consul-retry` + * Add `-consul-retry-attempts` + * Add `-consul-retry-backoff` + * Add `-vault-retry` + * Add `-vault-retry-attempts` + * Add `-vault-retry-backoff` + * Add support for `server_name` option for TLS configurations to allow + specification of the expected certificate common name. + * Add `-vault-addr` CLI option for specifying the Vault server address + [GH-740, GH-747] + * Add tagged addresses to Node structs + * Add support for multiple `-config` flags [GH-773, GH-751] + * Add more control over template command execution + * Add a way to programatically track the dependencies a particular template + is blocked on [GH-799] + +BUG FIXES: + + * Fix `-renew-token` flag not begin honored on the CLI [GH-741, GH-745] + * Allow `*` in key names [GH-789, GH-755] + +## v0.16.0 (September 22, 2016) + +NEW FEATURES: + + * **Exec Mode!** Consul Template can now act as a faux-supervisor for + applications. Please see the [Exec Mode](README.md#exec-mode) + documentation for more information. + * **Vault Token Unwrapping!** Consul Template can now unwrap Vault tokens that + have been wrapped using Vault's cubbyhole response wrapping. Simply add + the `unwrap_token` option to your Vault configuration stanza or pass in + the `-vault-unwrap-token` command line flag. + +BREAKING CHANGES: + + * Consul Template no longer terminates on SIGTERM or SIGQUIT. Previous + versions were hard-coded to listen for SIGINT, SIGTERM, and SIGQUIT. This + value is now configurable, and the default is SIGINT. SIGQUIT will trigger + a core dump in accordance with similar programs. SIGTERM is no longer + listened. + * Consul Template now exits on irrecoverable Vault errors such as failing to + renew a token or lease. + +DEPRECATIONS: + + * The `vault.renew` option has been renamed to `vault.renew_token` for added + clarity. This is backwards-compatible for this release, but will be + removed in a future release, so please update your configurations + accordingly. + +IMPROVEMENTS: + + * Permit commas in key prefix names [GH-669] + * Add configurable kill and reload signals [GH-686] + * Add a command line flag for controlling whether a provided Vault token will + be renewed [GH-718] + +BUG FIXES: + + * Allow variadic template function for `secret` [GH-660, GH-662] + * Always log in UTC time + * Log milliseconds [GH-676, GH-674] + * Maintain template ordering [GH-683] + * Add `Service` address to catalog node response [GH-687] + * Do not require trailing slashes [GH-706, GH-713] + * Wait for all existing dedup acquire attempts to finish [GH-716, GH-677] + + +## v0.15.0.dev (June 9, 2016) + +BREAKING CHANGES: + + * **Removing reaping functionality** [GH-628] + +IMPROVEMENTS: + + * Allow specifying per-template delimiters [GH-615, GH-389] + * Allow specifying per-template wait parameters [GH-589, GH-618] + * Switch to actually vendoring dependencies + * Add support for writing data [GH-652, GH-492] + +BUG FIXES: + + * Close open connections when reloading configuration [GH-591, GH-595] + * Do not share catalog nodes [GH-611, GH-572, GH-603] + * Properly handle empty string in ParseUint [GH-610, GH-609] + * Cache Vault's _original_ lease duration [5b955a8] + * Use decimal division for calculating Vault lease durations [87d61d9] + * Load VAULT_TOKEN environment variable [2431448] + * Properly clean up quiescence timers when using multiple templates [GH-616] + * Print a nice error if K/V cannot be exploded [GH-617, GH-596] + * Update documentation about symlinks [GH-579] + * Properly parse file permissions in mapstructure [GH-626] + +## v0.14.0 (March 7, 2016) + +DEPRECATIONS: + + * The `vault` template API function has been renamed to `secret` to be in line + with other tooling. The `vault` API function will continue to work but will + print a warning to the log file. A future release of Consul Template will + remove the `vault` API. + +NEW FEATURES: + + * Add `secrets` template API for listing secrets in Vault. Please note this + requires Vault 0.5+ and the secret backend must support listing. Please see + the Vault documentation for more information [GH-270] + +IMPROVEMENTS: + + * Allow passing any kind of object to `toJSON` in the template. Previously + this was restricted to key-value maps, but that restriction is now removed. + [GH-553] + +BUG FIXES: + + * Parse file permissions as a string in JSON [GH-548] + * Document how to reload config with signals [GH-522] + * Stop all dependencies when reloading the running/watcher [GH-534, GH-568] + +## v0.13.0 (February 18, 2016) + +BUG FIXES: + + * Compile with go1.6 to avoid race [GH-442] + * Switch to using a pooled transport [GH-546] + +## v0.12.2 (January 15, 2016) + +BUG FIXES: + + * Fixed an issue when running as PID 1 in a Docker container where Consul + Template could consume CPU and spuriously think its spwaned sub-processes + had failed [GH-511] + +## v0.12.1 (January 7, 2016) + +IMPROVEMENTS: + + * Add support for math operations on uint types [GH-483, GH-484] + * Make check information available through health service [GH-490] + +BUG FIXES: + + * Store vault data on the dependency and handle an error where a failed + lease renewal would result in `` in the rendered template. Please + note, there is a bug in Vault 0.4 with respect to lease renewals that makes + it inoperable with Consul Template. Please either use Vault 0.3 or wait + until Vault 0.5 is released (the bug has already been fixed on master). + [GH-468, GH-493, GH-504] + + +## v0.12.0 (December 10, 2015) + +BREAKING CHANGES: + + * Add support for checking if a node is in maintenance mode [GH-477, GH-455] + + Previously, Consul Template would report nodes in maintenance mode as + "critical". They will now report as "maintenance" so users can perform more + detailed filtering. It is unlikely, but if you were filtering critical + services, nodes/services in maintenance mode will no longer be included. + + +FEATURES: + + * Add support for de-duplication mode. In de-duplication mode, Consul Template + uses leader election to elect one Consul Template process to render a + template. The results of this template are rendered into Consul's key-value + store, and other templates pull from the pre-rendered template. This option + is off by default, but it is highly recommended that the option is enabled + for clusters with a high load factor (number of templates x number of + dependencies per template). [GH-465] + * Add support for automatically reaping child processes. This is very useful + when running Consul Template as PID 1 (like in a Docker container) when no + init system is present. The option is configurable, but it defaults to "on" + when the Consul Template process is PID 1. [GH-428, GH-479] + + +IMPROVEMENTS: + + * Use the `renew-self` endpoint instead of `renew` for renewing the token + [GH-450] + * Allow existing templates to be backed up before writing the new one [GH-464] + * Add support for TLS/SSL mutual authentication [GH-448] + * Add support for checking if a node is in maintenance mode [GH-477, GH-455] + + +## v0.11.1 (October 26, 2015) + +FEATURES: + + * Accept "unix" as an argument to `timestamp` to generate a unix + timestamp [GH-422] + +IMPROVEMENTS: + + * Make `Path` a public field on the vault secret dependency so other libraries + can access it + +BUG FIXES: + + * Ensure there is a newline at the end of the version output + * Update README development instructions [GH-423] + * Adjust error messages so that data does not always "come from Consul" + * Fix race conditions in tests + * Update the `LastContact` value for non-Consul dependencies to always + return 0 [GH-432, GH-433] + * Always use `DefaultConfig()` in tests to find issues + * Fix broken math functions - previously add, subtract, multiply, and divide + for integers would perform the operation on only the first operand + [GH-430, GH-435] + * Renew the vault token based off of the auth, not the secret [GH-443] + * Remove noisy log message [GH-445] + + +## v0.11.0 (October 9, 2015) + +BREAKING CHANGES: + + * Allow configuration of destination file permissions [GH-415, GH-358] + + Previously, Consul Template would inspect the file at the destination path + and mirror those file permissions, if a file existed. If a file did not + exist, Consul Template would render the file with 0644 permissions. This was + acceptable behavior in a pre-Vault world, but now that Consul Template is + capable of rendering secrets, there is a desire for increased security. As + such, Consul Template **no longer mirrors existing destination file + permissions**. Instead, users can specify the file permissions in the + configuration file. Please see the README for examples. If you were + previously relying on an existing file's file permissions to enfore the + destination file permissions, you must switch to specifying the file + permissions in the configuration file. If you were not dependent on this + behavior, nothing has changed; the default value is still 0644. + +FEATURES: + + * Add `in` and `contains` functions for checking if a slice or array contains + a given value [GH-366] + * Add `add` function for calculating the sum of integers/floats + * Add `subtract` function for calculating the difference of integers/floats + * Add `multiply` function for calculating the product of integers/floats + * Add `divide` function for calculating the division of integers/floats + +IMPROVEMENTS: + + * Sort serivces by ID as well + * Add a mechanism for renewing the given Vault token [GH-359, GH-367] + * Default max-stale to 1s - this severely reduces the load on the Consul + leader by allowing followers to respond to API requests [GH-386, GH-397] + * Add GPG signing for SHASUMS on new releases + * Push watcher errors down to the client in `once` mode [GH-361, GH-418] + +BUG FIXES: + + * Set ssl in the CLI [GH-321] + * **Regression** - Reload configuration on SIGHUP [GH-332] + * Remove port option from `service` query and documentation - it was unused + and legacy, but was causing issues and confusion [GH-333] + * Return the empty value when no parsable value is given [GH-353] + * Start with a blank configuration when reloading via SIGHUP [GH-393, GH-394] + * Use an int64 instead of an int to loop function [GH-401, GH-402] + * Do not remove the Windows file if it exists [GH-378] + +## v0.10.0 (June 9, 2015) + +FEATURES: + + * Add `plugin` and plugin ecosystem + * Add `parseBool` function for parsing strings into booleans (GH-312) + * Add `parseFloat` function for parsing strings into float64 (GH-312) + * Add `parseInt` function for parsing strings into int64 (GH-312) + * Add `parseUint` function for parsing strings into uint64 (GH-312) + * Add `explode` function for exploding the result of `tree` or `ls` into a + deeply nested hash (GH-311) + * Add `toJSON` and `toJSONPretty` function for exporting the result of `tree` + or `ls` into a JSON hash (GH-311) + * Add `toYAML` function for exporting the result of `tree` or `ls` into a + YAML document (GH-311) + * Add `node` function for querying nodes (GH-306, GH-309) + * Add `split` function for splitting a string on a separator (GH-285) + * Add `join` function for joining a string slice on a given key (GH-285) + * Add `pid_file` configuration and command line option for specifying the + location of a pid file on disk (GH-281, GH-286) + +IMPROVEMENTS: + + * Allow setting log_level via the configuration file (CLI still take + precedence if specified) + * Improve error reporting when loading multiple configs by including the path + on the configuration file that had an error (GH-275) + * Add a timeout around command execution to prevent hanging (GH-283) + * Read Vault/Consul environment variables for the config (GH-307, GH-308) + +BUG FIXES: + + * Properly merge "default" config values with user-supplied values (GH-271) + + +## v0.9.0 (April 29, 2015) + +FEATURES: + + * Add Vault functionality for querying secrets from Vault (GH-264) + * Add `regexMatch` template helper to determine if a result matches the given + regular expressions (GH-246) + * Add support for `ssl-cert` and `ss-ca-cert` options (GH-255) + +IMPROVEMENTS: + + * Expand `byTag` to accept catalog services as well (GH-249, GH-250) + * Allow catalog service tags to use the `.Contains` function (GH-261) + +BUG FIXES: + + * Send the standard error of commands back over the standard error of + Consul Template (GH-253, GH-254) + * Allow specifying `-v` in addition to `-version` to get the version output + +## v0.8.0 (March 30, 2015) + +FEATURES: + + * Add `.Size()` so the watcher can report its size (GH-206) + * Add `byKey` template helper to group the results of a `tree` function by + their containing directory (GH-207, GH-209, GH-241) + * Add `timestamp` template function for returning the current timestamp with + the ability to add custom formatting (GH-225, GH-230) + * Add `loop` template function for iteration (GH-238, GH-221) + +IMPROVEMENTS: + + * Expose `LastIndex` and `ReceivedData` from the Watcher + * Add unimplemented KV fields (GH-203) + * Warn the user if there are a large number of dependencies (GH-205) + * Extend documentation on how health service dependencies are downloaded from + Consul (GH-212) + * Allow empty configuration directories (GH-217) + * Document caveats around using `parseJSON` during multi-evaluation + * Print the final configuration as JSON in debug mode (GH-231) + * Export certain environment variables when executing commands that are read + by other Consul tooling or in your scripts (GH-232) - see the README for + more information + * Adjust logging to be less noisy without compromising information (GH-242) + +BUG FIXES: + + * Properly filter services by their type (GH-210, GH-212) + * Return an error if extra arguments are given on the command line (GH-227) + * Do not overwrite given configuration with the default options (GH-228, GH-219) + * Check for the correct conditions when using basic authentication (GH-220) + * Remove unused code paths for clarity (GH-242) + * Remove race condition in templates when called concurrently (GH-242) + * Remove race condition in test suite (GH-242) + * Force a refresh if Consul's WaitIndex is less than our current value (GH-242) + * Avoid pushing data onto the watcher when the view has been stopped (GH-242) + * Do not accept data in the runner for an unwatched dependency (GH-198, GH-242) + +## v0.7.0 (February 19, 2015) + +BREAKING CHANGES: + + * Remove `ssl` configuration option from templates - use an `ssl` + configuration block with `enabled = true` instead + * Remove `ssl_no_verify` configuration option from templates - use an `ssl` + configuration block with `verify = false` instead + * Restructure CLI `-ssl-no-verify` to `-ssl-verify` - to disable SSL + certification validation on the command line, use `-ssl-verify=false` + * Remove `auth` configuration option from templates - use an `auth` + configuration block with `enabled = true` combined with `username = ...` and + `password = ...` inside the block instead + +FEATURES: + + * Add support for logging to syslog (GH-163) + * Add `log_level` as a configuration file option + * Add `-log-level` as a CLI option + +IMPROVEMENTS: + + * Use a default retry interval of 5s (GH-190) - this value has been (and will + remain) configurable since v0.5.0, but the default value has changed from 0 + to 5s + * Use a service's reported address if given (GH-185, GH-186) + * Add new `NodeAddress` field to health services to always include the node's + address + * Return errors up the watcher's error channel so other libraries can + determine what to do with the error instead of swallowing it (GH-196) + * Move SSL and authentication options into their own configuration blocks in + the HCL + * Add new `watch.WaitVar` for parsing Wait structs via Go's flag parsing + library. + * Extract logging components into their own library for sharing (GH-199) + +BUG FIXES: + + * Return errors instead of nil in catalog nodes and key prefix dependencies + (GH-192) + * Allow Consul Template to exit when running in `once` mode and templates have + not changed (GH-188) + * Raise an error when specifying a non-existent option in the configuration + file (GH-197) + * Use an RWLock when accessing information in the Brain to improve performance + * Improve debugging output and consistency + * Remove unused Brain functions + * Remove unused documentation items + * Use the correct default values for `-ssl` and `-retry` on the CLI + +## v0.6.5 (February 5, 2015) + +FEATURES: + + * Add `-max-stale` to specify Consul Template may talk to non-leader Consul + nodes if they are less than the maximum stale value (GH-183) + +BUG FIXES: + + * Fix a concurrency bug in the Brain (GH-180) + * Add a better queue-draining mechanism for templates that have a large number + of dependencies (GH-184) + +## v0.6.1 (February 2, 2015) + +IMPROVEMENTS: + + * Allow watcher to use buffered channels so we do not block when multiple + dependencies return data (GH-176) + * Buffer results from the watcher to reduce the number of CPU cycles (GH-168 + and GH-178) + +BUG FIXES: + + * Handle the case where reloading via SIGHUP would cause an error (GH-175 and + GH-177) + * Return errors to the template when parsing a key fails (GH-170) + * Expand the list of possible values for keys to non-ASCII fields (the `@` is + still a restricted character because it denotes the datacenter) (GH-170) + * Diff missing dependencies during the template render to avoid creating + extra watchers (GH-169) + * Improve debugging output (GH-169) + +## v0.6.0 (January 20, 2015) + +FEATURES: + + * Implement n-pass evaluation (GH-64) - templates are now evaluated N+1 times + to properly accumulate dependencies and build the graph properly + +BREAKING CHANGES: + + * Remove `storeKeyPrefix` template function - it has been replaced with `ls` + and/or `tree` and was deprecated in 0.2.0 + * Remove `Key()` from dependency interface + +IMPROVEMENTS: + + * Switch to using `hashicorp/consul/api` instead of `armon/consul-api` + * Add support for communicating with Consul via HTTPS/SSL (GH-143) + * Add support for communicating with Consul via BasicAuth (GH-147) + * Quiesce on a per-template basis + +BUG FIXES: + + * Reduce memory footprint when running with a large number of templates by + using a single context instead of separate template contexts for each + template + * Improve test coverage + * Improve debugging output + * Correct tag deep copy that could result in 2N-1 tags (GH-155) + * Return an empty slice when parsing an empty JSON file + * Update README documentation + +## v0.5.1 (December 25, 2014) + +BUG FIXES: + + * Parse Retry values in the config (GH-136) + * Remove `util` package as it is a code smell and separate `Watcher` and + `Dependency` structs and functions into their own packages for re-use + (GH-137) + +## v0.5.0 (December 19, 2014) + +FEATURES: + + * Reload configuration on `SIGHUP` + * Add `services` template function for listing all services and associated + tags in the Consul catalog (GH-77) + +BUG FIXES: + + * Do not execute the same command more than once in one run (GH-112) + * Do not exit when Consul is unavailable (GH-103) + * Accept configuration files as a valid option to `-config` (GH-126) + * Accept Windows drive letters in template paths (GH-78) + * Deep copy and sort data returned from Consul API (specifically tags) + * Run commands even if not all templates have received data (GH-119) + +IMPROVEMENTS: + + * Add support for more complex service health filtering (GH-116) + * Add support for specifying a `-retry` interval for Consul timeouts and + connection errors (GH-22) + * Use official HashiCorp multierror package for errors + * Gracefully stop watchers on interrupt + * Add support for Go 1.4 + * Improve test coverage around retrying failures + +## v0.4.0 (December 10, 2014) + +FEATURES: + + * Add `env` template function for reading an environment variable in the + current process into the template + * Add `regexReplaceAll` template function + +BUG FIXES: + + * Fix documentation examples + * Fix `golint` and `go vet` errors + * Fix a panic when Consul returned empty query metadata + * Allow colons in key prefixes (`ls` and `tree` receive this by proxy) + * Allow `parseJSON` to handle top-level JSON objects + * Filter empty keys in `tree` and `ls` (folder nodes) + +IMPROVEMENTS: + + * Merge multiple configuration template definitions when a configuration + directory is specified + +## v0.3.1 (November 24, 2014) + +BUG FIXES: + + * Allow colons in key names (GH-67) + * Fix a documentation bug in the README in the Varnish example (GH-82) + * Attempt to render templates before starting the watcher - this fixes an + issue where a template that declared no Consul dependencies would never be + rendered (GH-85) + * Update inline Go documentation for better clarity + +IMPROVEMENTS: + + * Fix all issues raised by `go vet` + * Update packaging script to fix ZSHisms and use awk for clarity + +## v0.3.0 (November 13, 2014) + +FEATURES: + + * Added a `Contains` method to `Service.Tags` + * Added support for specifying a configuration directory in `-config`, in + addition to a file + * Added support for querying all nodes in Consul's catalog with the `nodes` + template function + +BUG FIXES: + + * Update README documentation to clarify that `service` dependencies default + to the current datacenter if one is not explicitly given + * Ignore empty keys that are returned from an `ls` call (GH-54) + * When writing a file atomicly, ensure the drive is the same (GH-58) + * Run all commands before exiting - previously if a single command failed in + a multi-template environment, the other commands would not execute, but + Consul Template would return + +IMPROVEMENTS: + + * Added support for querying all `service` nodes by passing an additional + parameter to `service` + +## v0.2.0 (November 4, 2014) + +FEATURES: + + * Added helper for decoding a result as JSON using the `parseJSON` pipe + function + * Added support for reading and watching changes from a file using the `file` + template function + * Added helper for sorting service entires by a particular tag + * Added helper function `toLower()` for converting a string to lowercase + * Added helper function `toTitle()` for converting a string to titlecase + * Added helper function `toUpper()` for converting a string to uppercase + * Added helper function `replaceAll()` for replacing occurrences of a + substring with a new string + * Added `tree` function for returning all key prefixes recursively + * Added `ls` function for returning all keys in the top-level prefix (but not + deeply nested ones) + +BUG FIXES: + + * Remove prefixes from paths when querying a key prefix + +IMPROVEMENTS: + + * Moved shareable functions into a util module so other libraries can benefit + * Make Path a public field on Template + * Added more examples and documentation to the README + +DEPRECATIONS: + + * `keyPrefix` is deprecated in favor or `tree` and `ls` and will be removed in + the next major release + + +## v0.1.1 (October 28, 2014) + +BUG FIXES: + + * Fixed an issue where help output was displayed twice when specifying the + `-h` flag + * Added support for specifyiny forward slashes (`/`) in service names + * Added support for specifying underscores (`_`) in service names + * Added support for specifying dots (`.`) in tag names + +IMPROVEMENTS: + + * Added support for Travis CI + * Fixed numerous typographical errors + * Added more documentation, including an FAQ in the README + * Do not return an error when a template has no dependencies. See GH-31 for + more background and information + * Do not render templates if they have the same content + * Do not execute commands if the template on disk would not be changed + +## v0.1.0 (October 21, 2014) + + * Initial release diff --git a/vendor/github.com/hashicorp/consul-template/Gopkg.lock b/vendor/github.com/hashicorp/consul-template/Gopkg.lock new file mode 100644 index 00000000000..114d6ff4b4a --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/Gopkg.lock @@ -0,0 +1,696 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:7202718ddfaa07d3c88e6d7bee854aa2ddceea5c75fa74c6c9f33de4db677ece" + name = "github.com/Jeffail/gabs" + packages = ["."] + pruneopts = "" + revision = "2a3aa15961d5fee6047b8151b67ac2f08ba2c48c" + version = "1.0" + +[[projects]] + digest = "1:b0fe84bcee1d0c3579d855029ccd3a76deea187412da2976985e4946289dbb2c" + name = "github.com/NYTimes/gziphandler" + packages = ["."] + pruneopts = "" + revision = "2600fb119af974220d3916a5916d6e31176aac1b" + version = "v1.0.1" + +[[projects]] + digest = "1:8855efc2aff3afd6319da41b22a8ca1cfd1698af05a24852c01636ba65b133f0" + name = "github.com/SermoDigital/jose" + packages = [ + ".", + "crypto", + "jws", + "jwt", + ] + pruneopts = "" + revision = "f6df55f235c24f236d11dbcf665249a59ac2021f" + version = "1.1" + +[[projects]] + branch = "master" + digest = "1:a96de7a26ef8bf2eccf3c5fc8039455b0259b19af1a91b7749afd674e3971efa" + name = "github.com/armon/go-metrics" + packages = ["."] + pruneopts = "" + revision = "9a4b6e10bed6220a1665955aa2b75afc91eb10b3" + +[[projects]] + branch = "master" + digest = "1:2a1e6af234d7de1ccf4504f397cf7cfa82922ee59b29252e3c34cb38d0b91989" + name = "github.com/armon/go-radix" + packages = ["."] + pruneopts = "" + revision = "1fca145dffbcaa8fe914309b1ec0cfc67500fe61" + +[[projects]] + digest = "1:62fe5a93293c353dafe321ad07419b680257596b0886fc5d21cd1fd42ad8ef45" + name = "github.com/asaskevich/govalidator" + packages = ["."] + pruneopts = "" + revision = "73945b6115bfbbcc57d89b7316e28109364124e1" + version = "v7" + +[[projects]] + digest = "1:289dd4d7abfb3ad2b5f728fbe9b1d5c1bf7d265a3eb9ef92869af1f7baba4c7a" + name = "github.com/burntsushi/toml" + packages = ["."] + pruneopts = "" + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" + +[[projects]] + digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "" + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + digest = "1:044b2f1eea2f5cfb0d3678baf60892734f59d5c2ea3932cb6ed894a97ccba15c" + name = "github.com/elazarl/go-bindata-assetfs" + packages = ["."] + pruneopts = "" + revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43" + version = "v1.0.0" + +[[projects]] + digest = "1:55848e643a99a9dfceb19e090ce67111328fbb1780f34c62a0430994ff85fb90" + name = "github.com/fatih/structs" + packages = ["."] + pruneopts = "" + revision = "a720dfa8df582c51dee1b36feabb906bde1588bd" + version = "v1.0" + +[[projects]] + digest = "1:24f8932912fd9331367d38715bb74be889dc2f94d401109c3aa3db8b3aa246c5" + name = "github.com/go-sql-driver/mysql" + packages = ["."] + pruneopts = "" + revision = "a0583e0143b1624142adab07e0e97fe106d99561" + version = "v1.3" + +[[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp", + ] + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + branch = "master" + digest = "1:09307dfb1aa3f49a2bf869dcfa4c6c06ecd3c207221bd1c1a1141f0e51f209eb" + name = "github.com/golang/snappy" + packages = ["."] + pruneopts = "" + revision = "553a641470496b2327abcac10b36396bd98e45c9" + +[[projects]] + branch = "master" + digest = "1:355da89acb2e3ee7342821708e2d1d51b29487e0642f5356282c42e2a9d3763f" + name = "github.com/hashicorp/consul" + packages = [ + "api", + "lib/freeport", + "testutil", + "testutil/retry", + ] + pruneopts = "" + revision = "73e3252076f69a06386a98a528bb79fa43bd538e" + +[[projects]] + branch = "master" + digest = "1:304c322b62533a48ac052ffee80f67087fce1bc07186cd4e610a1b0e77765836" + name = "github.com/hashicorp/errwrap" + packages = ["."] + pruneopts = "" + revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" + +[[projects]] + digest = "1:05334858a0cfb538622a066e065287f63f42bee26a7fda93a789674225057201" + name = "github.com/hashicorp/go-cleanhttp" + packages = ["."] + pruneopts = "" + revision = "e8ab9daed8d1ddd2d3c4efba338fe2eeae2e4f18" + version = "v0.5.0" + +[[projects]] + branch = "master" + digest = "1:504ef443922ff6f9e03d00babe3ac6c2fcb44f4fe6244c82cbb77d7ca76fdd87" + name = "github.com/hashicorp/go-gatedio" + packages = ["."] + pruneopts = "" + revision = "8b8de1022221dde1fb52fa25d0caab46e59c8c14" + +[[projects]] + branch = "master" + digest = "1:0b41d818c95c27c2618eef67569afb6356c3e55d7e8459fdf21ed015884f83ef" + name = "github.com/hashicorp/go-hclog" + packages = ["."] + pruneopts = "" + revision = "4783caec6f2e5cdd47fab8b2bb47ce2ce5c546b7" + +[[projects]] + branch = "master" + digest = "1:6546c6d83de55dc47f3211e82d1e588baeb432e33859ccb1195ce52890466053" + name = "github.com/hashicorp/go-immutable-radix" + packages = ["."] + pruneopts = "" + revision = "8aac2701530899b64bdea735a1de8da899815220" + +[[projects]] + branch = "master" + digest = "1:7b4ee3a9138e3757a0238f2e12b97e2c6a33a5b9230386ea99692a56f6f0bc2a" + name = "github.com/hashicorp/go-memdb" + packages = ["."] + pruneopts = "" + revision = "032f93b25becbfd6c3bb074a1049d98b7e105440" + +[[projects]] + branch = "master" + digest = "1:7660b6ee3fd92bcb9b19f5d359d3fbc8e853257d8a3d49e0424d00b6faa69cfd" + name = "github.com/hashicorp/go-multierror" + packages = ["."] + pruneopts = "" + revision = "83588e72410abfbe4df460eeb6f30841ae47d4c4" + +[[projects]] + branch = "master" + digest = "1:2474b03b87dbe1274652da5541e18fec7125107fcd5a83d5928d1616f851394c" + name = "github.com/hashicorp/go-plugin" + packages = [ + ".", + "internal/proto", + ] + pruneopts = "" + revision = "362c99b11937c6a84686ee5726a8170e921ab406" + +[[projects]] + digest = "1:776139dc18d63ef223ffaca5d8e9a3057174890f84393d3c881e934100b66dbc" + name = "github.com/hashicorp/go-retryablehttp" + packages = ["."] + pruneopts = "" + revision = "73489d0a1476f0c9e6fb03f9c39241523a496dfd" + version = "v0.5.2" + +[[projects]] + branch = "master" + digest = "1:ff65bf6fc4d1116f94ac305342725c21b55c16819c2606adc8f527755716937f" + name = "github.com/hashicorp/go-rootcerts" + packages = ["."] + pruneopts = "" + revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + +[[projects]] + digest = "1:ea71015bc8aa9b98a1fde564e24123260330b55bda32d8ed5ce227d3dc58d64e" + name = "github.com/hashicorp/go-sockaddr" + packages = ["."] + pruneopts = "" + revision = "3aed17b5ee41761cc2b04f2a94c7107d428967e5" + version = "v1.0.1" + +[[projects]] + branch = "master" + digest = "1:4d9d876a856ada3b553062ac8e50331a9a539e12893c0c4a50d8ae2af4242685" + name = "github.com/hashicorp/go-syslog" + packages = ["."] + pruneopts = "" + revision = "326bf4a7f709d263f964a6a96558676b103f3534" + +[[projects]] + branch = "master" + digest = "1:50518e39c832eacbbd55a0ca08c6490911fe9483b06fa77468693a31b7893f3e" + name = "github.com/hashicorp/go-uuid" + packages = ["."] + pruneopts = "" + revision = "64130c7a86d732268a38cb04cfbaf0cc987fda98" + +[[projects]] + digest = "1:b759103c9b4135568253c17d2866064cde398e93764b611caabf5aa8e3059685" + name = "github.com/hashicorp/go-version" + packages = ["."] + pruneopts = "" + revision = "d40cf49b3a77bba84a7afdbd7f1dc295d114efb1" + version = "v1.1.0" + +[[projects]] + branch = "master" + digest = "1:43987212a2f16bfacc1a286e9118f212d60c136ed53c6c9477c18921db53140b" + name = "github.com/hashicorp/golang-lru" + packages = [ + ".", + "simplelru", + ] + pruneopts = "" + revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6" + +[[projects]] + branch = "master" + digest = "1:147d671753effde6d3bcd58fc74c1d67d740196c84c280c762a5417319499972" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token", + ] + pruneopts = "" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + +[[projects]] + branch = "master" + digest = "1:8b7dd3b581147b44cf522c66894b9119ab845c346d8f124d83f77ab499cf7ca3" + name = "github.com/hashicorp/logutils" + packages = ["."] + pruneopts = "" + revision = "0dc08b1671f34c4250ce212759ebd880f743d883" + +[[projects]] + digest = "1:f72168ea995f398bab88e84bd1ff58a983466ba162fb8d50d47420666cd57fad" + name = "github.com/hashicorp/serf" + packages = ["coordinate"] + pruneopts = "" + revision = "d6574a5bb1226678d7010325fb6c985db20ee458" + version = "v0.8.1" + +[[projects]] + branch = "master" + digest = "1:ac8c3b2c00d263ab59323ef42ca6a85145447a4fd8c9dd661d88156fe7efe006" + name = "github.com/hashicorp/vault" + packages = [ + "api", + "audit", + "builtin/logical/database/dbplugin", + "builtin/logical/pki", + "builtin/logical/transit", + "builtin/plugin", + "helper/base62", + "helper/certutil", + "helper/compressutil", + "helper/consts", + "helper/cryptoutil", + "helper/dbtxn", + "helper/errutil", + "helper/forwarding", + "helper/hclutil", + "helper/identity", + "helper/identity/mfa", + "helper/jsonutil", + "helper/kdf", + "helper/keysutil", + "helper/license", + "helper/locksutil", + "helper/logging", + "helper/mlock", + "helper/namespace", + "helper/parseutil", + "helper/pathmanager", + "helper/pgpkeys", + "helper/pluginutil", + "helper/policyutil", + "helper/reload", + "helper/salt", + "helper/storagepacker", + "helper/strutil", + "helper/tlsutil", + "helper/wrapping", + "helper/xor", + "http", + "logical", + "logical/framework", + "logical/plugin", + "logical/plugin/pb", + "physical", + "physical/inmem", + "plugins", + "plugins/database/mysql", + "plugins/database/postgresql", + "plugins/helper/database/connutil", + "plugins/helper/database/credsutil", + "plugins/helper/database/dbutil", + "shamir", + "vault", + "vault/seal", + "version", + ] + pruneopts = "" + revision = "be968f0edd5991df4237ab184ab94ea649d15b43" + +[[projects]] + branch = "master" + digest = "1:18f7a8c6df80b7ad85be744c6b6334983539896350d77760e90d8462ff51be6d" + name = "github.com/hashicorp/vault-plugin-secrets-kv" + packages = ["."] + pruneopts = "" + revision = "edbfe287c5d9277cecf2c91c79ffcc34f19d2049" + +[[projects]] + branch = "master" + digest = "1:755f0590df531fdf5221158ba457555b525ea497f27ae1b8b195da7d0906d4a6" + name = "github.com/hashicorp/yamux" + packages = ["."] + pruneopts = "" + revision = "f5742cb6b85602e7fa834e9d5d91a7d7fa850824" + +[[projects]] + branch = "master" + digest = "1:5d8602d6ebb444e0c18792d61fd4bb302a0d4d0b02cebf50c475f9dbeaabb884" + name = "github.com/jefferai/jsonx" + packages = ["."] + pruneopts = "" + revision = "9cc31c3135eef39b8e72585f37efa92b6ca314d0" + +[[projects]] + branch = "master" + digest = "1:ad122173a3e31da3986e097c26422fe9c765899e2afdf86eeca1ec360e57eff9" + name = "github.com/keybase/go-crypto" + packages = [ + "brainpool", + "cast5", + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "openpgp", + "openpgp/armor", + "openpgp/ecdh", + "openpgp/elgamal", + "openpgp/errors", + "openpgp/packet", + "openpgp/s2k", + "rsa", + ] + pruneopts = "" + revision = "f63716704117f5bd34d8f0f068f7e8369d20d4ab" + +[[projects]] + branch = "master" + digest = "1:c7bbf42b56f999fc18f12707f6f9a3f47171de8bc6d4d7d3e8449093d55a4629" + name = "github.com/lib/pq" + packages = [ + ".", + "oid", + ] + pruneopts = "" + revision = "b609790bd85edf8e9ab7e0f8912750a786177bcf" + +[[projects]] + digest = "1:477cce5379198d3b8230b5c0961c61fcd1b337371cda81318e89a109245d83cb" + name = "github.com/mattn/go-shellwords" + packages = ["."] + pruneopts = "" + revision = "02e3cf038dcea8290e44424da473dd12be796a8a" + version = "v1.0.3" + +[[projects]] + branch = "master" + digest = "1:ae14aee05347b333fd7ab0c801c789438ef559cfb1307b53d5c42ea3cf6d61b6" + name = "github.com/mitchellh/copystructure" + packages = ["."] + pruneopts = "" + revision = "d23ffcb85de31694d6ccaa23ccb4a03e55c1303f" + +[[projects]] + branch = "master" + digest = "1:59d11e81d6fdd12a771321696bb22abdd9a94d26ac864787e98c9b419e428734" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + pruneopts = "" + revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + +[[projects]] + branch = "master" + digest = "1:51c98e2c9a8d0a724a69f46421876af14e12132cb02f1d0e144785d752247162" + name = "github.com/mitchellh/go-testing-interface" + packages = ["."] + pruneopts = "" + revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" + +[[projects]] + branch = "master" + digest = "1:0de0f377aeccd41384e883c59c6f184c9db01c96db33a2724a1eaadd60f92629" + name = "github.com/mitchellh/hashstructure" + packages = ["."] + pruneopts = "" + revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" + +[[projects]] + branch = "master" + digest = "1:30a2adc78c422ebd23aac9cfece529954d5eacf9ddbe37345f2a17439f8fa849" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + pruneopts = "" + revision = "06020f85339e21b2478f756a78e295255ffa4d6a" + +[[projects]] + branch = "master" + digest = "1:a5aebbd13aa160140a1fd1286b94cd8c6ba3d1522014fd04508d7f36d5bb8d19" + name = "github.com/mitchellh/reflectwalk" + packages = ["."] + pruneopts = "" + revision = "63d60e9d0dbc60cf9164e6510889b0db6683d98c" + +[[projects]] + digest = "1:94e9081cc450d2cdf4e6886fc2c06c07272f86477df2d74ee5931951fa3d2577" + name = "github.com/oklog/run" + packages = ["."] + pruneopts = "" + revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39" + version = "v1.0.0" + +[[projects]] + digest = "1:4c0404dc03d974acd5fcd8b8d3ce687b13bd169db032b89275e8b9d77b98ce8c" + name = "github.com/patrickmn/go-cache" + packages = ["."] + pruneopts = "" + revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0" + version = "v2.1.0" + +[[projects]] + digest = "1:a1d7aa6caa82465a50a4c1da6f8dc9ff2ab4624a41b7020ef3d1fbed9ba9845d" + name = "github.com/pierrec/lz4" + packages = [ + ".", + "internal/xxh32", + ] + pruneopts = "" + revision = "473cd7ce01a1113208073166464b98819526150e" + version = "v2.0.8" + +[[projects]] + digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "" + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:29df111893b87bd947307aab294c042e900c2f29c53ad3896127955b4283728a" + name = "github.com/ryanuber/go-glob" + packages = ["."] + pruneopts = "" + revision = "572520ed46dbddaed19ea3d9541bdd0494163693" + version = "v0.1" + +[[projects]] + digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159" + name = "github.com/stretchr/testify" + packages = ["assert"] + pruneopts = "" + revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" + version = "v1.1.4" + +[[projects]] + branch = "master" + digest = "1:ee71b3559000aca2562869f53fe762743e84bf4f0993240d4c2e37f9122a032b" + name = "golang.org/x/crypto" + packages = [ + "blake2b", + "chacha20poly1305", + "cryptobyte", + "cryptobyte/asn1", + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "hkdf", + "internal/chacha20", + "internal/subtle", + "poly1305", + "ssh", + ] + pruneopts = "" + revision = "193df9c0f06f8bb35fba505183eaf0acc0136505" + +[[projects]] + branch = "master" + digest = "1:e3fd71c3687fb1d263e491fc3bd9013858aeb30a6393fc9b77cbbdc37d0f9727" + name = "golang.org/x/net" + packages = [ + "context", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "lex/httplex", + "trace", + ] + pruneopts = "" + revision = "c73622c77280266305273cb545f54516ced95b93" + +[[projects]] + branch = "master" + digest = "1:489610147902fe0c7229218c749bb25a8a9ecce0d726ae4f8662517319f32554" + name = "golang.org/x/sys" + packages = [ + "cpu", + "unix", + ] + pruneopts = "" + revision = "41f3e6584952bb034a481797859f6ab34b6803bd" + +[[projects]] + branch = "master" + digest = "1:bf8bd584b40670bc7e4a50bde42e87ede902ab048c84b2d1710aab4d76dac7a1" + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable", + ] + pruneopts = "" + revision = "6eab0e8f74e86c598ec3b6fad4888e0c11482d48" + +[[projects]] + branch = "master" + digest = "1:14cb1d4240bcbbf1386ae763957e04e2765ec4e4ce7bb2769d05fa6faccd774e" + name = "golang.org/x/time" + packages = ["rate"] + pruneopts = "" + revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd" + +[[projects]] + branch = "master" + digest = "1:180913ea45cbe0072abce387a686b929908f8213106a735fe1d1273ae5239648" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + pruneopts = "" + revision = "f676e0f3ac6395ff1a529ae59a6670878a8371a6" + +[[projects]] + digest = "1:39d4d828b87d58d114fdc211f0638f32dcae84019fe17d6b48f9b697f4b60213" + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", + "codes", + "connectivity", + "credentials", + "credentials/internal", + "encoding", + "encoding/proto", + "grpclog", + "health", + "health/grpc_health_v1", + "internal", + "internal/backoff", + "internal/binarylog", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + ] + pruneopts = "" + revision = "a02b0774206b209466313a0b525d2c738fe407eb" + version = "v1.18.0" + +[[projects]] + branch = "v2" + digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61" + name = "gopkg.in/yaml.v2" + packages = ["."] + pruneopts = "" + revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/burntsushi/toml", + "github.com/hashicorp/consul/api", + "github.com/hashicorp/consul/testutil", + "github.com/hashicorp/go-gatedio", + "github.com/hashicorp/go-hclog", + "github.com/hashicorp/go-multierror", + "github.com/hashicorp/go-rootcerts", + "github.com/hashicorp/go-syslog", + "github.com/hashicorp/hcl", + "github.com/hashicorp/logutils", + "github.com/hashicorp/vault-plugin-secrets-kv", + "github.com/hashicorp/vault/api", + "github.com/hashicorp/vault/builtin/logical/pki", + "github.com/hashicorp/vault/builtin/logical/transit", + "github.com/hashicorp/vault/helper/namespace", + "github.com/hashicorp/vault/http", + "github.com/hashicorp/vault/logical", + "github.com/hashicorp/vault/physical/inmem", + "github.com/hashicorp/vault/vault", + "github.com/mattn/go-shellwords", + "github.com/mitchellh/go-homedir", + "github.com/mitchellh/hashstructure", + "github.com/mitchellh/mapstructure", + "github.com/pkg/errors", + "github.com/stretchr/testify/assert", + "gopkg.in/yaml.v2", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/hashicorp/consul-template/Gopkg.toml b/vendor/github.com/hashicorp/consul-template/Gopkg.toml new file mode 100644 index 00000000000..8ebdd4f5bd9 --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/Gopkg.toml @@ -0,0 +1,64 @@ + +[[constraint]] + name = "github.com/burntsushi/toml" + version = "0.3.0" + +[[constraint]] + name = "github.com/hashicorp/consul" + branch = "master" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-gatedio" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-multierror" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-rootcerts" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-syslog" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/hcl" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/logutils" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/vault" + +[[constraint]] + name = "github.com/mattn/go-shellwords" + version = "1.0.3" + +[[constraint]] + name = "github.com/mgutz/logxi" + version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/mitchellh/go-homedir" + +[[constraint]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + +[[constraint]] + name = "github.com/pkg/errors" + version = "0.8.0" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.1.4" + +[[constraint]] + branch = "v2" + name = "gopkg.in/yaml.v2" diff --git a/vendor/github.com/hashicorp/consul-template/Makefile b/vendor/github.com/hashicorp/consul-template/Makefile new file mode 100644 index 00000000000..5adbad1a8f4 --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/Makefile @@ -0,0 +1,236 @@ +# Metadata about this makefile and position +MKFILE_PATH := $(lastword $(MAKEFILE_LIST)) +CURRENT_DIR := $(patsubst %/,%,$(dir $(realpath $(MKFILE_PATH)))) + +# Ensure GOPATH +GOPATH ?= $(HOME)/go + +# List all our actual files, excluding vendor +GOFILES ?= $(shell go list $(TEST) | grep -v /vendor/) + +# Tags specific for building +GOTAGS ?= + +# Number of procs to use +GOMAXPROCS ?= 4 + +# Get the project metadata +GOVERSION := 1.11.5 +PROJECT := $(CURRENT_DIR:$(GOPATH)/src/%=%) +OWNER := $(notdir $(patsubst %/,%,$(dir $(PROJECT)))) +NAME := $(notdir $(PROJECT)) +GIT_COMMIT ?= $(shell git rev-parse --short HEAD) +VERSION := $(shell awk -F\" '/Version/ { print $$2; exit }' "${CURRENT_DIR}/version/version.go") +EXTERNAL_TOOLS = \ + github.com/golang/dep/cmd/dep + +# Current system information +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) + +# Default os-arch combination to build +XC_OS ?= darwin freebsd linux netbsd openbsd solaris windows +XC_ARCH ?= 386 amd64 arm +XC_EXCLUDE ?= darwin/arm solaris/386 solaris/arm windows/arm + +# GPG Signing key (blank by default, means no GPG signing) +GPG_KEY ?= + +# List of ldflags +LD_FLAGS ?= \ + -s \ + -w \ + -X ${PROJECT}/version.Name=${NAME} \ + -X ${PROJECT}/version.GitCommit=${GIT_COMMIT} + +# List of Docker targets to build +DOCKER_TARGETS ?= alpine scratch + +# List of tests to run +TEST ?= ./... + +# Create a cross-compile target for every os-arch pairing. This will generate +# a make target for each os/arch like "make linux/amd64" as well as generate a +# meta target (build) for compiling everything. +define make-xc-target + $1/$2: + ifneq (,$(findstring ${1}/${2},$(XC_EXCLUDE))) + @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT} (excluded)" + else + @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT}" + @docker run \ + --interactive \ + --rm \ + --dns="8.8.8.8" \ + --volume="${CURRENT_DIR}:/go/src/${PROJECT}" \ + --workdir="/go/src/${PROJECT}" \ + "golang:${GOVERSION}" \ + env \ + CGO_ENABLED="0" \ + GOOS="${1}" \ + GOARCH="${2}" \ + go build \ + -a \ + -o="pkg/${1}_${2}/${NAME}${3}" \ + -ldflags "${LD_FLAGS}" \ + -tags "${GOTAGS}" + endif + .PHONY: $1/$2 + + $1:: $1/$2 + .PHONY: $1 + + build:: $1/$2 + .PHONY: build +endef +$(foreach goarch,$(XC_ARCH),$(foreach goos,$(XC_OS),$(eval $(call make-xc-target,$(goos),$(goarch),$(if $(findstring windows,$(goos)),.exe,))))) + +# bootstrap installs the necessary go tools for development or build. +bootstrap: + @echo "==> Bootstrapping ${PROJECT}" + @for t in ${EXTERNAL_TOOLS}; do \ + echo "--> Installing $$t" ; \ + go get -u "$$t"; \ + done +.PHONY: bootstrap + +# deps updates all dependencies for this project. +deps: + @echo "==> Updating deps for ${PROJECT}" + @dep ensure -update + @dep prune +.PHONY: deps + +# dev builds and installs the project locally. +dev: + @echo "==> Installing ${NAME} for ${GOOS}/${GOARCH}" + @rm -f "${GOPATH}/pkg/${GOOS}_${GOARCH}/${PROJECT}/version.a" # ldflags change and go doesn't detect + @env \ + CGO_ENABLED="0" \ + go install \ + -ldflags "${LD_FLAGS}" \ + -tags "${GOTAGS}" +.PHONY: dev + +# dist builds the binaries and then signs and packages them for distribution +dist: +ifndef GPG_KEY + @echo "==> ERROR: No GPG key specified! Without a GPG key, this release cannot" + @echo " be signed. Set the environment variable GPG_KEY to the ID of" + @echo " the GPG key to continue." + @exit 127 +else + @$(MAKE) -f "${MKFILE_PATH}" _cleanup + @$(MAKE) -f "${MKFILE_PATH}" -j4 build + @$(MAKE) -f "${MKFILE_PATH}" _compress _checksum _sign +endif +.PHONY: dist + +# Create a docker compile and push target for each container. This will create +# docker-build/scratch, docker-push/scratch, etc. It will also create two meta +# targets: docker-build and docker-push, which will build and push all +# configured Docker containers. Each container must have a folder in docker/ +# named after itself with a Dockerfile (docker/alpine/Dockerfile). +define make-docker-target + docker-build/$1: + @echo "==> Building ${1} Docker container for ${PROJECT}" + @docker build \ + --rm \ + --force-rm \ + --no-cache \ + --squash \ + --compress \ + --file="docker/${1}/Dockerfile" \ + --build-arg="LD_FLAGS=${LD_FLAGS}" \ + --build-arg="GOTAGS=${GOTAGS}" \ + $(if $(filter $1,scratch),--tag="${OWNER}/${NAME}",) \ + --tag="${OWNER}/${NAME}:${1}" \ + --tag="${OWNER}/${NAME}:${VERSION}-${1}" \ + "${CURRENT_DIR}" + .PHONY: docker-build/$1 + + docker-build:: docker-build/$1 + .PHONY: docker-build + + docker-push/$1: + @echo "==> Pushing ${1} to Docker registry" + $(if $(filter $1,scratch),@docker push "${OWNER}/${NAME}",) + @docker push "${OWNER}/${NAME}:${1}" + @docker push "${OWNER}/${NAME}:${VERSION}-${1}" + .PHONY: docker-push/$1 + + docker-push:: docker-push/$1 + .PHONY: docker-push +endef +$(foreach target,$(DOCKER_TARGETS),$(eval $(call make-docker-target,$(target)))) + +# test runs the test suite. +test: + @echo "==> Testing ${NAME}" + @go test -timeout=30s -parallel=20 -tags="${GOTAGS}" ${GOFILES} ${TESTARGS} +.PHONY: test + +# test-race runs the test suite. +test-race: + @echo "==> Testing ${NAME} (race)" + @go test -timeout=60s -race -tags="${GOTAGS}" ${GOFILES} ${TESTARGS} +.PHONY: test-race + +# _cleanup removes any previous binaries +_cleanup: + @rm -rf "${CURRENT_DIR}/pkg/" + @rm -rf "${CURRENT_DIR}/bin/" + +# _compress compresses all the binaries in pkg/* as tarball and zip. +_compress: + @mkdir -p "${CURRENT_DIR}/pkg/dist" + @for platform in $$(find ./pkg -mindepth 1 -maxdepth 1 -type d); do \ + osarch=$$(basename "$$platform"); \ + if [ "$$osarch" = "dist" ]; then \ + continue; \ + fi; \ + \ + ext=""; \ + if test -z "$${osarch##*windows*}"; then \ + ext=".exe"; \ + fi; \ + cd "$$platform"; \ + tar -czf "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_$${osarch}.tgz" "${NAME}$${ext}"; \ + zip -q "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_$${osarch}.zip" "${NAME}$${ext}"; \ + cd - &>/dev/null; \ + done +.PHONY: _compress + +# _checksum produces the checksums for the binaries in pkg/dist +_checksum: + @cd "${CURRENT_DIR}/pkg/dist" && \ + shasum --algorithm 256 * > ${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_SHA256SUMS && \ + cd - &>/dev/null +.PHONY: _checksum + +# _sign signs the binaries using the given GPG_KEY. This should not be called +# as a separate function. +_sign: + @echo "==> Signing ${PROJECT} at v${VERSION}" + @gpg \ + --default-key "${GPG_KEY}" \ + --detach-sig "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_SHA256SUMS" + @git commit \ + --allow-empty \ + --gpg-sign="${GPG_KEY}" \ + --message "Release v${VERSION}" \ + --quiet \ + --signoff + @git tag \ + --annotate \ + --create-reflog \ + --local-user "${GPG_KEY}" \ + --message "Version ${VERSION}" \ + --sign \ + "v${VERSION}" master + @echo "--> Do not forget to run:" + @echo "" + @echo " git push && git push --tags" + @echo "" + @echo "And then upload the binaries in dist/!" +.PHONY: _sign diff --git a/vendor/github.com/hashicorp/consul-template/README.md b/vendor/github.com/hashicorp/consul-template/README.md new file mode 100644 index 00000000000..b337336753a --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/README.md @@ -0,0 +1,2214 @@ +# Consul Template + +[![Build Status](http://img.shields.io/travis/hashicorp/consul-template.svg?style=flat-square)](https://travis-ci.org/hashicorp/consul-template) +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/hashicorp/consul-template) + +This project provides a convenient way to populate values from [Consul][consul] +into the file system using the `consul-template` daemon. + +The daemon `consul-template` queries a [Consul][consul] or [Vault][vault] +cluster and updates any number of specified templates on the file system. As an +added bonus, it can optionally run arbitrary commands when the update process +completes. Please see the [examples folder][examples] for some scenarios where +this functionality might prove useful. + +--- + +**The documentation in this README corresponds to the master branch of Consul Template. It may contain unreleased features or different APIs than the most recently released version.** + +**Please see the [Git tag](https://github.com/hashicorp/consul-template/releases) that corresponds to your version of Consul Template for the proper documentation.** + +--- + + +## Installation + +1. Download a pre-compiled, released version from the [Consul Template releases page][releases]. + +1. Extract the binary using `unzip` or `tar`. + +1. Move the binary into `$PATH`. + +To compile from source, please see the instructions in the +[contributing section](#contributing). + +## Quick Example + +This short example assumes Consul is installed locally. + +1. Start a Consul cluster in dev mode: + + ```shell + $ consul agent -dev + ``` + +1. Author a template `in.tpl` to query the kv store: + + ```liquid + {{ key "foo" }} + ``` + +1. Start Consul Template: + + ```shell + $ consul-template -template "in.tpl:out.txt" -once + ``` + +1. Write data to the key in Consul: + + ```shell + $ consul kv put foo bar + ``` + +1. Observe Consul Template has written the file `out.txt`: + + ```shell + $ cat out.txt + bar + ``` + +For more examples and use cases, please see the [examples folder][examples] in +this repository. + +## Usage + +For the full list of options: + +```shell +$ consul-template -h +``` + +### Command Line Flags + +The CLI interface supports all options in the configuration file and vice-versa. Here are a few examples of common integrations on the command line. + +Render the template on disk at `/tmp/template.ctmpl` to `/tmp/result`: + +```shell +$ consul-template \ + -template "/tmp/template.ctmpl:/tmp/result" +``` + +Render multiple templates in the same process. The optional third argument to +the template is a command that will execute each time the template changes. + +```shell +$ consul-template \ + -template "/tmp/nginx.ctmpl:/var/nginx/nginx.conf:nginx -s reload" \ + -template "/tmp/redis.ctmpl:/var/redis/redis.conf:service redis restart" \ + -template "/tmp/haproxy.ctmpl:/var/haproxy/haproxy.conf" +``` + +Render a template using a custom Consul and Vault address: + +```shell +$ consul-template \ + -consul-addr "10.4.4.6:8500" \ + -vault-addr "https://10.5.32.5:8200" +``` + +Render all templates and then spawn and monitor a child process as a supervisor: + +```shell +$ consul-template \ + -template "/tmp/in.ctmpl:/tmp/result" \ + -exec "/sbin/my-server" +``` + +For more information on supervising, please see the +[Consul Template Exec Mode documentation](#exec-mode). + +### Configuration File Format + +Configuration files are written in the [HashiCorp Configuration Language][hcl]. +By proxy, this means the configuration is also JSON compatible. + +```hcl +# This denotes the start of the configuration section for Consul. All values +# contained in this section pertain to Consul. +consul { + # This block specifies the basic authentication information to pass with the + # request. For more information on authentication, please see the Consul + # documentation. + auth { + enabled = true + username = "test" + password = "test" + } + + # This is the address of the Consul agent. By default, this is + # 127.0.0.1:8500, which is the default bind and port for a local Consul + # agent. It is not recommended that you communicate directly with a Consul + # server, and instead communicate with the local Consul agent. There are many + # reasons for this, most importantly the Consul agent is able to multiplex + # connections to the Consul server and reduce the number of open HTTP + # connections. Additionally, it provides a "well-known" IP address for which + # clients can connect. + address = "127.0.0.1:8500" + + # This is the ACL token to use when connecting to Consul. If you did not + # enable ACLs on your Consul cluster, you do not need to set this option. + # + # This option is also available via the environment variable CONSUL_TOKEN. + token = "abcd1234" + + # This controls the retry behavior when an error is returned from Consul. + # Consul Template is highly fault tolerant, meaning it does not exit in the + # face of failure. Instead, it uses exponential back-off and retry functions + # to wait for the cluster to become available, as is customary in distributed + # systems. + retry { + # This enabled retries. Retries are enabled by default, so this is + # redundant. + enabled = true + + # This specifies the number of attempts to make before giving up. Each + # attempt adds the exponential backoff sleep time. Setting this to + # zero will implement an unlimited number of retries. + attempts = 12 + + # This is the base amount of time to sleep between retry attempts. Each + # retry sleeps for an exponent of 2 longer than this base. For 5 retries, + # the sleep times would be: 250ms, 500ms, 1s, 2s, then 4s. + backoff = "250ms" + + # This is the maximum amount of time to sleep between retry attempts. + # When max_backoff is set to zero, there is no upper limit to the + # exponential sleep between retry attempts. + # If max_backoff is set to 10s and backoff is set to 1s, sleep times + # would be: 1s, 2s, 4s, 8s, 10s, 10s, ... + max_backoff = "1m" + } + + # This block configures the SSL options for connecting to the Consul server. + ssl { + # This enables SSL. Specifying any option for SSL will also enable it. + enabled = true + + # This enables SSL peer verification. The default value is "true", which + # will check the global CA chain to make sure the given certificates are + # valid. If you are using a self-signed certificate that you have not added + # to the CA chain, you may want to disable SSL verification. However, please + # understand this is a potential security vulnerability. + verify = false + + # This is the path to the certificate to use to authenticate. If just a + # certificate is provided, it is assumed to contain both the certificate and + # the key to convert to an X509 certificate. If both the certificate and + # key are specified, Consul Template will automatically combine them into an + # X509 certificate for you. + cert = "/path/to/client/cert" + key = "/path/to/client/key" + + # This is the path to the certificate authority to use as a CA. This is + # useful for self-signed certificates or for organizations using their own + # internal certificate authority. + ca_cert = "/path/to/ca" + + # This is the path to a directory of PEM-encoded CA cert files. If both + # `ca_cert` and `ca_path` is specified, `ca_cert` is preferred. + ca_path = "path/to/certs/" + + # This sets the SNI server name to use for validation. + server_name = "my-server.com" + } +} + +# This is the signal to listen for to trigger a reload event. The default +# value is shown below. Setting this value to the empty string will cause CT +# to not listen for any reload signals. +reload_signal = "SIGHUP" + +# This is the signal to listen for to trigger a graceful stop. The default +# value is shown below. Setting this value to the empty string will cause CT +# to not listen for any graceful stop signals. +kill_signal = "SIGINT" + +# This is the maximum interval to allow "stale" data. By default, only the +# Consul leader will respond to queries; any requests to a follower will +# forward to the leader. In large clusters with many requests, this is not as +# scalable, so this option allows any follower to respond to a query, so long +# as the last-replicated data is within these bounds. Higher values result in +# less cluster load, but are more likely to have outdated data. +max_stale = "10m" + +# This is the log level. If you find a bug in Consul Template, please enable +# debug logs so we can help identify the issue. This is also available as a +# command line flag. +log_level = "warn" + +# This is the path to store a PID file which will contain the process ID of the +# Consul Template process. This is useful if you plan to send custom signals +# to the process. +pid_file = "/path/to/pid" + +# This is the quiescence timers; it defines the minimum and maximum amount of +# time to wait for the cluster to reach a consistent state before rendering a +# template. This is useful to enable in systems that have a lot of flapping, +# because it will reduce the the number of times a template is rendered. +wait { + min = "5s" + max = "10s" +} + +# This denotes the start of the configuration section for Vault. All values +# contained in this section pertain to Vault. +vault { + # This is the address of the Vault leader. The protocol (http(s)) portion + # of the address is required. + address = "https://vault.service.consul:8200" + + # This is the grace period between lease renewal of periodic secrets and secret + # re-acquisition. When renewing a secret, if the remaining lease is less than or + # equal to the configured grace, Consul Template will request a new credential. + # This prevents Vault from revoking the credential at expiration and Consul + # Template having a stale credential. + # + # Note: If you set this to a value that is higher than your default TTL or + # max TTL, Consul Template will always read a new secret! + # + # This should also be less than or around 1/3 of your TTL for a predictable + # behaviour. See https://github.com/hashicorp/vault/issues/3414 + grace = "5m" + + # This is a Vault Enterprise namespace to use for reading/writing secrets. + # + # This value can also be specified via the environment variable VAULT_NAMESPACE. + namespace = "foo" + + # This is the token to use when communicating with the Vault server. + # Like other tools that integrate with Vault, Consul Template makes the + # assumption that you provide it with a Vault token; it does not have the + # incorporated logic to generate tokens via Vault's auth methods. + # + # This value can also be specified via the environment variable VAULT_TOKEN. + # When using a token from Vault Agent, the vault_agent_token_file setting + # should be used instead, as that will take precedence over this field. + token = "abcd1234" + + # This tells Consul Template to load the Vault token from the contents of a file. + # If this field is specified: + # - Consul Template will not try to renew the Vault token. + # - Consul Template will periodically stat the file and update the token if it has + # changed. + # vault_agent_token_file = "/tmp/vault/agent/token" + + # This tells Consul Template that the provided token is actually a wrapped + # token that should be unwrapped using Vault's cubbyhole response wrapping + # before being used. Please see Vault's cubbyhole response wrapping + # documentation for more information. + unwrap_token = true + + # This option tells Consul Template to automatically renew the Vault token + # given. If you are unfamiliar with Vault's architecture, Vault requires + # tokens be renewed at some regular interval or they will be revoked. Consul + # Template will automatically renew the token at half the lease duration of + # the token. The default value is true, but this option can be disabled if + # you want to renew the Vault token using an out-of-band process. + # + # Note that secrets specified in a template (using {{secret}} for example) + # are always renewed, even if this option is set to false. This option only + # applies to the top-level Vault token itself. + renew_token = true + + # This section details the retry options for connecting to Vault. Please see + # the retry options in the Consul section for more information (they are the + # same). + retry { + # ... + } + + # This section details the SSL options for connecting to the Vault server. + # Please see the SSL options in the Consul section for more information (they + # are the same). + ssl { + # ... + } +} + +# This block defines the configuration for connecting to a syslog server for +# logging. +syslog { + # This enables syslog logging. Specifying any other option also enables + # syslog logging. + enabled = true + + # This is the name of the syslog facility to log to. + facility = "LOCAL5" +} + +# This block defines the configuration for de-duplication mode. Please see the +# de-duplication mode documentation later in the README for more information +# on how de-duplication mode operates. +deduplicate { + # This enables de-duplication mode. Specifying any other options also enables + # de-duplication mode. + enabled = true + + # This is the prefix to the path in Consul's KV store where de-duplication + # templates will be pre-rendered and stored. + prefix = "consul-template/dedup/" +} + +# This block defines the configuration for exec mode. Please see the exec mode +# documentation at the bottom of this README for more information on how exec +# mode operates and the caveats of this mode. +exec { + # This is the command to exec as a child process. There can be only one + # command per Consul Template process. + command = "/usr/bin/app" + + # This is a random splay to wait before killing the command. The default + # value is 0 (no wait), but large clusters should consider setting a splay + # value to prevent all child processes from reloading at the same time when + # data changes occur. When this value is set to non-zero, Consul Template + # will wait a random period of time up to the splay value before reloading + # or killing the child process. This can be used to prevent the thundering + # herd problem on applications that do not gracefully reload. + splay = "5s" + + env { + # This specifies if the child process should not inherit the parent + # process's environment. By default, the child will have full access to the + # environment variables of the parent. Setting this to true will send only + # the values specified in `custom_env` to the child process. + pristine = false + + # This specifies additional custom environment variables in the form shown + # below to inject into the child's runtime environment. If a custom + # environment variable shares its name with a system environment variable, + # the custom environment variable takes precedence. Even if pristine, + # whitelist, or blacklist is specified, all values in this option + # are given to the child process. + custom = ["PATH=$PATH:/etc/myapp/bin"] + + # This specifies a list of environment variables to exclusively include in + # the list of environment variables exposed to the child process. If + # specified, only those environment variables matching the given patterns + # are exposed to the child process. These strings are matched using Go's + # glob function, so wildcards are permitted. + whitelist = ["CONSUL_*"] + + # This specifies a list of environment variables to exclusively prohibit in + # the list of environment variables exposed to the child process. If + # specified, any environment variables matching the given patterns will not + # be exposed to the child process, even if they are whitelisted. The values + # in this option take precedence over the values in the whitelist. + # These strings are matched using Go's glob function, so wildcards are + # permitted. + blacklist = ["VAULT_*"] + } + + # This defines the signal that will be sent to the child process when a + # change occurs in a watched template. The signal will only be sent after the + # process is started, and the process will only be started after all + # dependent templates have been rendered at least once. The default value is + # nil, which tells Consul Template to stop the child process and spawn a new + # one instead of sending it a signal. This is useful for legacy applications + # or applications that cannot properly reload their configuration without a + # full reload. + reload_signal = "" + + # This defines the signal sent to the child process when Consul Template is + # gracefully shutting down. The application should begin a graceful cleanup. + # If the application does not terminate before the `kill_timeout`, it will + # be terminated (effectively "kill -9"). The default value is "SIGTERM". + kill_signal = "SIGINT" + + # This defines the amount of time to wait for the child process to gracefully + # terminate when Consul Template exits. After this specified time, the child + # process will be force-killed (effectively "kill -9"). The default value is + # "30s". + kill_timeout = "2s" +} + +# This block defines the configuration for a template. Unlike other blocks, +# this block may be specified multiple times to configure multiple templates. +# It is also possible to configure templates via the CLI directly. +template { + # This is the source file on disk to use as the input template. This is often + # called the "Consul Template template". This option is required if not using + # the `contents` option. + source = "/path/on/disk/to/template.ctmpl" + + # This is the destination path on disk where the source template will render. + # If the parent directories do not exist, Consul Template will attempt to + # create them, unless create_dest_dirs is false. + destination = "/path/on/disk/where/template/will/render.txt" + + # This options tells Consul Template to create the parent directories of the + # destination path if they do not exist. The default value is true. + create_dest_dirs = true + + # This option allows embedding the contents of a template in the configuration + # file rather then supplying the `source` path to the template file. This is + # useful for short templates. This option is mutually exclusive with the + # `source` option. + contents = "{{ keyOrDefault \"service/redis/maxconns@east-aws\" \"5\" }}" + + # This is the optional command to run when the template is rendered. The + # command will only run if the resulting template changes. The command must + # return within 30s (configurable), and it must have a successful exit code. + # Consul Template is not a replacement for a process monitor or init system. + command = "restart service foo" + + # This is the maximum amount of time to wait for the optional command to + # return. Default is 30s. + command_timeout = "60s" + + # Exit with an error when accessing a struct or map field/key that does not + # exist. The default behavior will print "" when accessing a field + # that does not exist. It is highly recommended you set this to "true" when + # retrieving secrets from Vault. + error_on_missing_key = false + + # This is the permission to render the file. If this option is left + # unspecified, Consul Template will attempt to match the permissions of the + # file that already exists at the destination path. If no file exists at that + # path, the permissions are 0644. + perms = 0600 + + # This option backs up the previously rendered template at the destination + # path before writing a new one. It keeps exactly one backup. This option is + # useful for preventing accidental changes to the data without having a + # rollback strategy. + backup = true + + # These are the delimiters to use in the template. The default is "{{" and + # "}}", but for some templates, it may be easier to use a different delimiter + # that does not conflict with the output file itself. + left_delimiter = "{{" + right_delimiter = "}}" + + # This is the `minimum(:maximum)` to wait before rendering a new template to + # disk and triggering a command, separated by a colon (`:`). If the optional + # maximum value is omitted, it is assumed to be 4x the required minimum value. + # This is a numeric time with a unit suffix ("5s"). There is no default value. + # The wait value for a template takes precedence over any globally-configured + # wait. + wait { + min = "2s" + max = "10s" + } +} + +``` + +Note that not all fields are required. If you are not retrieving secrets from +Vault, you do not need to specify a Vault configuration section. Similarly, if +you are not logging to syslog, you do not need to specify a syslog +configuration. + +For additional security, tokens may also be read from the environment using the +`CONSUL_TOKEN` or `VAULT_TOKEN` environment variables respectively. It is highly +recommended that you do not put your tokens in plain-text in a configuration +file. + +Instruct Consul Template to use a configuration file with the `-config` flag: + +```shell +$ consul-template -config "/my/config.hcl" +``` + +This argument may be specified multiple times to load multiple configuration +files. The right-most configuration takes the highest precedence. If the path to +a directory is provided (as opposed to the path to a file), all of the files in +the given directory will be merged in +[lexical order](http://golang.org/pkg/path/filepath/#Walk), recursively. Please +note that symbolic links are _not_ followed. + +**Commands specified on the CLI take precedence over a config file!** + +### Templating Language + +Consul Template parses files authored in the [Go Template][text-template] +format. If you are not familiar with the syntax, please read Go's documentation +and examples. In addition to the Go-provided template functions, Consul Template +provides the following functions: + +#### API Functions + +API functions interact with remote API calls, communicating with external +services like [Consul][consul] and [Vault][vault]. + +##### `datacenters` + +Query [Consul][consul] for all datacenters in its catalog. + +```liquid +{{ datacenters }} +``` + +For example: + +```liquid +{{ range datacenters }} +{{ . }}{{ end }} +``` + +renders + +```text +dc1 +dc2 +``` + +An optional boolean can be specified which instructs Consul Template to ignore +datacenters which are inaccessible or do not have a current leader. Enabling +this option requires an O(N+1) operation and therefore is not recommended in +environments where performance is a factor. + +```liquid +// Ignores datacenters which are inaccessible +{{ datacenters true }} +``` + +##### `file` + +Read and output the contents of a local file on disk. If the file cannot be +read, an error will occur. When the file changes, Consul Template will pick up +the change and re-render the template. + +```liquid +{{ file "" }} +``` + +For example: + +```liquid +{{ file "/path/to/my/file" }} +``` + +renders + +```text +file contents +``` + +This does not process nested templates. See +[`executeTemplate`](#executeTemplate) for a way to render nested templates. + +##### `key` + +Query [Consul][consul] for the value at the given key path. If the key does not +exist, Consul Template will block rendering until the key is present. To avoid +blocking, use `keyOrDefault` or `keyExists`. + +```liquid +{{ key "@" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ key "service/redis/maxconns" }} +``` + +renders + +```text +15 +``` + +##### `keyExists` + +Query [Consul][consul] for the value at the given key path. If the key exists, +this will return true, false otherwise. Unlike `key`, this function will not +block if the key does not exist. This is useful for controlling flow. + +```liquid +{{ keyExists "@" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ if keyExists "app/beta_active" }} + # ... +{{ else }} + # ... +{{ end }} +``` + +##### `keyOrDefault` + +Query [Consul][consul] for the value at the given key path. If the key does not +exist, the default value will be used instead. Unlike `key`, this function will +not block if the key does not exist. + +```liquid +{{ keyOrDefault "@" "" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ keyOrDefault "service/redis/maxconns" "5" }} +``` + +renders + +```text +5 +``` + +Note that Consul Template uses a [multi-phase +execution](#multi-phase-execution). During the first phase of evaluation, Consul +Template will have no data from Consul and thus will _always_ fall back to the +default value. Subsequent reads from Consul will pull in the real value from +Consul (if the key exists) on the next template pass. This is important because +it means that Consul Template will never "block" the rendering of a template due +to a missing key from a `keyOrDefault`. Even if the key exists, if Consul has +not yet returned data for the key, the default value will be used instead. + +##### `ls` + +Query [Consul][consul] for all top-level kv pairs at the given key path. + +```liquid +{{ ls "@" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ range ls "service/redis" }} +{{ .Key }}:{{ .Value }}{{ end }} +``` + +renders + +```text +maxconns:15 +minconns:5 +``` + +##### `node` + +Query [Consul][consul] for a node in the catalog. + +```liquid +{{node "@"}} +``` + +The `` attribute is optional; if omitted, the local agent node is used. + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ with node }} +{{ .Node.Address }}{{ end }} +``` + +renders + +```text +10.5.2.6 +``` + +To query a different node: + +```liquid +{{ with node "node1@dc2" }} +{{ .Node.Address }}{{ end }} +``` + +renders + +```text +10.4.2.6 +``` + +To access map data such as `TaggedAddresses` or `Meta`, use +[Go's text/template][text-template] map indexing. + +##### `nodes` + +Query [Consul][consul] for all nodes in the catalog. + +```liquid +{{ nodes "@~" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +The `` attribute is optional; if omitted, results are specified in lexical +order. If provided a node name, results are ordered by shortest round-trip time +to the provided node. If provided `_agent`, results are ordered by shortest +round-trip time to the local agent. + +For example: + +```liquid +{{ range nodes }} +{{ .Address }}{{ end }} +``` + +renders + +```text +10.4.2.13 +10.46.2.5 +``` + +To query a different data center and order by shortest trip time to ourselves: + +```liquid +{{ range nodes "@dc2~_agent" }} +{{ .Address }}{{ end }} +``` + +To access map data such as `TaggedAddresses` or `Meta`, use +[Go's text/template][text-template] map indexing. + +##### `secret` + +Query [Vault][vault] for the secret at the given path. + +```liquid +{{ secret "" "" }} +``` + +The `` attribute is optional; if omitted, the request will be a `vault +read` (HTTP GET) request. If provided, the request will be a `vault write` (HTTP +PUT/POST) request. + +For example: + +```liquid +{{ with secret "secret/passwords" }} +{{ .Data.wifi }}{{ end }} +``` + +renders + +```text +FORWARDSoneword +``` + +To access a versioned secret value (for the K/V version 2 backend): + +```liquid +{{ with secret "secret/passwords?version=1" }} +{{ .Data.data.wifi }}{{ end }} +``` + +When omitting the `?version` parameter, the latest version of the secret will be +fetched. Note the nested `.Data.data` syntax when referencing the secret value. +For more information about using the K/V v2 backend, see the +[Vault Documentation](https://www.vaultproject.io/docs/secrets/kv/kv-v2.html). + +When using Vault versions 0.10.0/0.10.1, the secret path will have to be prefixed +with "data", i.e. `secret/data/passwords` for the example above. This is not +necessary for Vault versions after 0.10.1, as consul-template will detect the KV +backend version being used. The version 2 KV backend did not exist prior to 0.10.0, +so these are the only affected versions. + +An example using write to generate PKI certificates: + +```liquid +{{ with secret "pki/issue/my-domain-dot-com" "common_name=foo.example.com" }} +{{ .Data.certificate }}{{ end }} +``` + +The parameters must be `key=value` pairs, and each pair must be its own argument +to the function: + +Please always consider the security implications of having the contents of a +secret in plain-text on disk. If an attacker is able to get access to the file, +they will have access to plain-text secrets. + +Please note that Vault does not support blocking queries. As a result, Consul +Template will not immediately reload in the event a secret is changed as it does +with Consul's key-value store. Consul Template will fetch a new secret at half +the lease duration of the original secret. For example, most items in Vault's +generic secret backend have a default 30 day lease. This means Consul Template +will renew the secret every 15 days. As such, it is recommended that a smaller +lease duration be used when generating the initial secret to force Consul +Template to renew more often. + +Also consider enabling `error_on_missing_key` when working with templates that +will interact with Vault. By default, Consul Template uses Go's templating +language. When accessing a struct field or map key that does not exist, it +defaults to printing "". This may not be the desired behavior, +especially when working with passwords or other data. As such, it is recommended +you set: + +```hcl +template { + error_on_missing_key = true +} +``` + +You can also guard against empty values using `if` or `with` blocks. + +```liquid +{{ with secret "secret/foo"}} +{{ if .Data.password }} +password = "{{ .Data.password }}" +{{ end }} +{{ end }} +``` + +##### `secrets` + +Query [Vault][vault] for the list of secrets at the given path. Not all +endpoints support listing. + +```liquid +{{ secrets "" }} +``` + +For example: + +```liquid +{{ range secrets "secret/" }} +{{ . }}{{ end }} +``` + +renders + +```text +bar +foo +zip +``` + +To iterate and list over every secret in the generic secret backend in Vault: + +```liquid +{{ range secrets "secret/" }} +{{ with secret (printf "secret/%s" .) }}{{ range $k, $v := .Data }} +{{ $k }}: {{ $v }} +{{ end }}{{ end }}{{ end }} +``` + +You should probably never do this. + +Please also note that Vault does not support +blocking queries. To understand the implications, please read the note at the +end of the `secret` function. + +##### `service` + +Query [Consul][consul] for services based on their health. + +```liquid +{{ service ".@~|" }} +``` + +The `` attribute is optional; if omitted, all nodes will be queried. + +The `` attribute is optional; if omitted, the local datacenter is +used. + +The `` attribute is optional; if omitted, results are specified in lexical +order. If provided a node name, results are ordered by shortest round-trip time +to the provided node. If provided `_agent`, results are ordered by shortest +round-trip time to the local agent. + +The `` attribute is optional; if omitted, only health services are +returned. Providing a filter allows for client-side filtering of services. + +For example: + +The example above is querying Consul for healthy "web" services, in the "east-aws" data center. The tag and data center attributes are optional. To query all nodes of the "web" service (regardless of tag) for the current data center: + +```liquid +{{ range service "web" }} +server {{ .Name }}{{ .Address }}:{{ .Port }}{{ end }} +``` + +renders the IP addresses of all _healthy_ nodes with a logical service named +"web": + +```text +server web01 10.5.2.45:2492 +server web02 10.2.6.61:2904 +``` + +To access map data such as `NodeTaggedAddresses` or `NodeMeta`, use +[Go's text/template][text-template] map indexing. + +By default only healthy services are returned. To list all services, pass the +"any" filter: + +```liquid +{{ service "web|any" }} +``` + +This will return all services registered to the agent, regardless of their +status. + +To filter services by a specific set of healths, specify a comma-separated list +of health statuses: + +```liquid +{{ service "web|passing,warning" }} +``` + +This will returns services which are deemed "passing" or "warning" according to +their node and service-level checks defined in Consul. Please note that the +comma implies an "or", not an "and". + +**Note:** There is an architectural difference between the following: + +```liquid +{{ service "web" }} +{{ service "web|passing" }} +``` + +The former will return all services which Consul considers "healthy" and +passing. The latter will return all services registered with the Consul agent +and perform client-side filtering. As a general rule, do not use the "passing" +argument alone if you want only healthy services - simply omit the second +argument instead. + +##### `services` + +Query [Consul][consul] for all services in the catalog. + +```liquid +{{ services "@" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ range services }} +{{ .Name }}: {{ .Tags | join "," }}{{ end }} +``` + +renders + +```text +node01 tag1,tag2,tag3 +``` + +##### `tree` + +Query [Consul][consul] for all kv pairs at the given key path. + +```liquid +{{ tree "@" }} +``` + +The `` attribute is optional; if omitted, the local datacenter is +used. + +For example: + +```liquid +{{ range tree "service/redis" }} +{{ .Key }}:{{ .Value }}{{ end }} +``` +renders + +```text +minconns 2 +maxconns 12 +nested/config/value "value" +``` + +Unlike `ls`, `tree` returns **all** keys under the prefix, just like the Unix +`tree` command. + +--- + +#### Scratch + +The scratchpad (or "scratch" for short) is available within the context of a +template to store temporary data or computations. Scratch data is not shared +between templates and is not cached between invocations. + +##### `scratch.Key` + +Returns a boolean if data exists in the scratchpad at the named key. Even if the +data at that key is "nil", this still returns true. + +```liquid +{{ scratch.Key "foo" }} +``` + +##### `scratch.Get` + +Returns the value in the scratchpad at the named key. If the data does not +exist, this will return "nil". + +```liquid +{{ scratch.Get "foo" }} +``` + +##### `scratch.Set` + +Saves the given value at the given key. If data already exists at that key, it +is overwritten. + +```liquid +{{ scratch.Set "foo" "bar" }} +``` + +##### `scratch.SetX` + +This behaves exactly the same as `Set`, but does not overwrite if the value +already exists. + +```liquid +{{ scratch.SetX "foo" "bar" }} +``` + +##### `scratch.MapSet` + +Saves a value in a named key in the map. If data already exists at that key, it +is overwritten. + +```liquid +{{ scratch.MapSet "vars" "foo" "bar" }} +``` + +##### `scratch.MapSetX` + +This behaves exactly the same as `MapSet`, but does not overwrite if the value +already exists. + +```liquid +{{ scratch.MapSetX "vars" "foo" "bar" }} +``` + +##### `scratch.MapValues` + +Returns a sorted list (by key) of all values in the named map. + +```liquid +{{ scratch.MapValues "vars" }} +``` + +--- + +#### Helper Functions + +Unlike API functions, helper functions do not query remote services. These +functions are useful for parsing data, formatting data, performing math, etc. + +##### `base64Decode` + +Accepts a base64-encoded string and returns the decoded result, or an error if +the given string is not a valid base64 string. + +```liquid +{{ base64Decode "aGVsbG8=" }} +``` + +renders + +```text +hello +``` + +##### `base64Encode` + +Accepts a string and returns a base64-encoded string. + +```liquid +{{ base64Encode "hello" }} +``` + +renders + +```text +aGVsbG8= +``` + +##### `base64URLDecode` + +Accepts a base64-encoded URL-safe string and returns the decoded result, or an +error if the given string is not a valid base64 URL-safe string. + +```liquid +{{ base64URLDecode "aGVsbG8=" }} +``` + +renders + +```text +hello +``` + +##### `base64URLEncode` + +Accepts a string and returns a base-64 encoded URL-safe string. + +```liquid +{{ base64Encode "hello" }} +``` + +renders + +```text +aGVsbG8= +``` + +##### `byKey` + +Accepts a list of pairs returned from a [`tree`](#tree) call and creates a map that groups pairs by their top-level directory. + +For example: + +```text +groups/elasticsearch/es1 +groups/elasticsearch/es2 +groups/elasticsearch/es3 +services/elasticsearch/check_elasticsearch +services/elasticsearch/check_indexes +``` + +with the following template + +```liquid +{{ range $key, $pairs := tree "groups" | byKey }}{{ $key }}: +{{ range $pair := $pairs }} {{ .Key }}={{ .Value }} +{{ end }}{{ end }} +``` + +renders + +```text +elasticsearch: + es1=1 + es2=1 + es3=1 +``` + +Note that the top-most key is stripped from the Key value. Keys that have no +prefix after stripping are removed from the list. + +The resulting pairs are keyed as a map, so it is possible to look up a single +value by key: + +```liquid +{{ $weights := tree "weights" }} +{{ range service "release.web" }} + {{ $weight := or (index $weights .Node) 100 }} + server {{ .Node }} {{ .Address }}:{{ .Port }} weight {{ $weight }}{{ end }} +``` + +##### `byTag` + +Takes the list of services returned by the [`service`](#service) or +[`services`](#services) function and creates a map that groups services by tag. + +```liquid +{{ range $tag, $services := service "web" | byTag }}{{ $tag }} +{{ range $services }} server {{ .Name }} {{ .Address }}:{{ .Port }} +{{ end }}{{ end }} +``` + +##### `contains` + +Determines if a needle is within an iterable element. + +```liquid +{{ if .Tags | contains "production" }} +# ... +{{ end }} +``` + +#### `containsAll` + +Returns `true` if all needles are within an iterable element, or `false` +otherwise. Returns `true` if the list of needles is empty. + +```liquid +{{ if containsAll $requiredTags .Tags }} +# ... +{{ end }} +``` + +#### `containsAny` + +Returns `true` if any needle is within an iterable element, or `false` +otherwise. Returns `false` if the list of needles is empty. + +```liquid +{{ if containsAny $acceptableTags .Tags }} +# ... +{{ end }} +``` + +#### `containsNone` + +Returns `true` if no needles are within an iterable element, or `false` +otherwise. Returns `true` if the list of needles is empty. + +```liquid +{{ if containsNone $forbiddenTags .Tags }} +# ... +{{ end }} +``` + +#### `containsNotAll` + +Returns `true` if some needle is not within an iterable element, or `false` +otherwise. Returns `false` if the list of needles is empty. + +```liquid +{{ if containsNotAll $excludingTags .Tags }} +# ... +{{ end }} +``` + +##### `env` + +Reads the given environment variable accessible to the current process. + +```liquid +{{ env "CLUSTER_ID" }} +``` + +This function can be chained to manipulate the output: + +```liquid +{{ env "CLUSTER_ID" | toLower }} +``` + +Reads the given environment variable and if it does not exist or is blank use a default value, ex `12345`. + +```liquid +{{ or (env "CLUSTER_ID") "12345" }} +``` + +##### `executeTemplate` + +Executes and returns a defined template. + +```liquid +{{ define "custom" }}my custom template{{ end }} + +This is my other template: +{{ executeTemplate "custom" }} + +And I can call it multiple times: +{{ executeTemplate "custom" }} + +Even with a new context: +{{ executeTemplate "custom" 42 }} + +Or save it to a variable: +{{ $var := executeTemplate "custom" }} +``` + +##### `explode` + +Takes the result from a `tree` or `ls` call and converts it into a deeply-nested +map for parsing/traversing. + +```liquid +{{ tree "config" | explode }} +``` + +Note: You will lose any metadata about the key-pair after it has been exploded. +You can also access deeply nested values: + +```liquid +{{ with tree "config" | explode }} +{{ .a.b.c }}{{ end }} +``` + +You will need to have a reasonable format about your data in Consul. Please see +[Go's text/template package][text-template] for more information. + + +##### `indent` + +Indents a block of text by prefixing N number of spaces per line. + +```liquid +{{ tree "foo" | explode | toYAML | indent 4 }} +``` + +##### `in` + +Determines if a needle is within an iterable element. + +```liquid +{{ if in .Tags "production" }} +# ... +{{ end }} +``` + +##### `loop` + +Accepts varying parameters and differs its behavior based on those parameters. + +If `loop` is given one integer, it will return a goroutine that begins at zero +and loops up to but not including the given integer: + +```liquid +{{ range loop 5 }} +# Comment{{end}} +``` + +If given two integers, this function will return a goroutine that begins at +the first integer and loops up to but not including the second integer: + +```liquid +{{ range $i := loop 5 8 }} +stanza-{{ $i }}{{ end }} +``` + +which would render: + +```text +stanza-5 +stanza-6 +stanza-7 +``` + +Note: It is not possible to get the index and the element since the function +returns a goroutine, not a slice. In other words, the following is **not +valid**: + +```liquid +# Will NOT work! +{{ range $i, $e := loop 5 8 }} +# ...{{ end }} +``` + +##### `join` + +Takes the given list of strings as a pipe and joins them on the provided string: + +```liquid +{{ $items | join "," }} +``` + +##### `trimSpace` + +Takes the provided input and trims all whitespace, tabs and newlines: + +```liquid +{{ file "/etc/ec2_version" | trimSpace }} +``` + +##### `parseBool` + +Takes the given string and parses it as a boolean: + +```liquid +{{ "true" | parseBool }} +``` + +This can be combined with a key and a conditional check, for example: + +```liquid +{{ if key "feature/enabled" | parseBool }}{{ end }} +``` + +##### `parseFloat` + +Takes the given string and parses it as a base-10 float64: + +```liquid +{{ "1.2" | parseFloat }} +``` + +##### `parseInt` + +Takes the given string and parses it as a base-10 int64: + +```liquid +{{ "1" | parseInt }} +``` + +This can be combined with other helpers, for example: + +```liquid +{{ range $i := loop key "config/pool_size" | parseInt }} +# ...{{ end }} +``` + +##### `parseJSON` + +Takes the given input (usually the value from a key) and parses the result as +JSON: + +```liquid +{{ with $d := key "user/info" | parseJSON }}{{ $d.name }}{{ end }} +``` + +Note: Consul Template evaluates the template multiple times, and on the first +evaluation the value of the key will be empty (because no data has been loaded +yet). This means that templates must guard against empty responses. + +##### `parseUint` + +Takes the given string and parses it as a base-10 int64: + +```liquid +{{ "1" | parseUint }} +``` + +##### `plugin` + +Takes the name of a plugin and optional payload and executes a Consul Template +plugin. + +```liquid +{{ plugin "my-plugin" }} +``` + +This is most commonly combined with a JSON filter for customization: + +```liquid +{{ tree "foo" | explode | toJSON | plugin "my-plugin" }} +``` + +Please see the [plugins](#plugins) section for more information about plugins. + +##### `regexMatch` + +Takes the argument as a regular expression and will return `true` if it matches +on the given string, or `false` otherwise. + +```liquid +{{ if "foo.bar" | regexMatch "foo([.a-z]+)" }} +# ... +{{ else }} +# ... +{{ end }} +``` + +##### `regexReplaceAll` + +Takes the argument as a regular expression and replaces all occurrences of the +regex with the given string. As in go, you can use variables like $1 to refer to +subexpressions in the replacement string. + +```liquid +{{ "foo.bar" | regexReplaceAll "foo([.a-z]+)" "$1" }} +``` + +##### `replaceAll` + +Takes the argument as a string and replaces all occurrences of the given string +with the given string. + +```liquid +{{ "foo.bar" | replaceAll "." "_" }} +``` + +This function can be chained with other functions as well: + +```liquid +{{ service "web" }}{{ .Name | replaceAll ":" "_" }}{{ end }} +``` + +##### `split` + +Splits the given string on the provided separator: + +```liquid +{{ "foo\nbar\n" | split "\n" }} +``` + +This can be combined with chained and piped with other functions: + +```liquid +{{ key "foo" | toUpper | split "\n" | join "," }} +``` + +##### `timestamp` + +Returns the current timestamp as a string (UTC). If no arguments are given, the +result is the current RFC3339 timestamp: + +```liquid +{{ timestamp }} // e.g. 1970-01-01T00:00:00Z +``` + +If the optional parameter is given, it is used to format the timestamp. The +magic reference date **Mon Jan 2 15:04:05 -0700 MST 2006** can be used to format +the date as required: + +```liquid +{{ timestamp "2006-01-02" }} // e.g. 1970-01-01 +``` + +See [Go's `time.Format`](http://golang.org/pkg/time/#Time.Format) for more +information. + +As a special case, if the optional parameter is `"unix"`, the unix timestamp in +seconds is returned as a string. + +```liquid +{{ timestamp "unix" }} // e.g. 0 +``` + +##### `toJSON` + +Takes the result from a `tree` or `ls` call and converts it into a JSON object. + +```liquid +{{ tree "config" | explode | toJSON }} +``` + +renders + +```javascript +{"admin":{"port":"1234"},"maxconns":"5","minconns":"2"} +``` + +Note: Consul stores all KV data as strings. Thus true is "true", 1 is "1", etc. + +##### `toJSONPretty` + +Takes the result from a `tree` or `ls` call and converts it into a +pretty-printed JSON object, indented by two spaces. + +```liquid +{{ tree "config" | explode | toJSONPretty }} +``` + +renders + +```javascript +{ + "admin": { + "port": "1234" + }, + "maxconns": "5", + "minconns": "2", +} +``` + +Note: Consul stores all KV data as strings. Thus true is "true", 1 is "1", etc. + +##### `toLower` + +Takes the argument as a string and converts it to lowercase. + +```liquid +{{ key "user/name" | toLower }} +``` + +See [Go's `strings.ToLower`](http://golang.org/pkg/strings/#ToLower) for more +information. + +##### `toTitle` + +Takes the argument as a string and converts it to titlecase. + +```liquid +{{ key "user/name" | toTitle }} +``` + +See [Go's `strings.Title`](http://golang.org/pkg/strings/#Title) for more +information. + +##### `toTOML` + +Takes the result from a `tree` or `ls` call and converts it into a TOML object. + +```liquid +{{ tree "config" | explode | toTOML }} +``` + +renders + +```toml +maxconns = "5" +minconns = "2" + +[admin] + port = "1134" +``` + +Note: Consul stores all KV data as strings. Thus true is "true", 1 is "1", etc. + +##### `toUpper` + +Takes the argument as a string and converts it to uppercase. + +```liquid +{{ key "user/name" | toUpper }} +``` + +See [Go's `strings.ToUpper`](http://golang.org/pkg/strings/#ToUpper) for more +information. + +##### `toYAML` + +Takes the result from a `tree` or `ls` call and converts it into a +pretty-printed YAML object, indented by two spaces. + +```liquid +{{ tree "config" | explode | toYAML }} +``` + +renders + +```yaml +admin: + port: "1234" +maxconns: "5" +minconns: "2" +``` + +Note: Consul stores all KV data as strings. Thus true is "true", 1 is "1", etc. + +--- + +#### Math Functions + +The following functions are available on floats and integer values. + +##### `add` + +Returns the sum of the two values. + +```liquid +{{ add 1 2 }} // 3 +``` + +This can also be used with a pipe function. + +```liquid +{{ 1 | add 2 }} // 3 +``` + +##### `subtract` + +Returns the difference of the second value from the first. + +```liquid +{{ subtract 2 5 }} // 3 +``` + +This can also be used with a pipe function. + +```liquid +{{ 5 | subtract 2 }} // 3 +``` + +Please take careful note of the order of arguments. + +##### `multiply` + +Returns the product of the two values. + +```liquid +{{ multiply 2 2 }} // 4 +``` + +This can also be used with a pipe function. + +```liquid +{{ 2 | multiply 2 }} // 4 +``` + +##### `divide` + +Returns the division of the second value from the first. + +```liquid +{{ divide 2 10 }} // 5 +``` + +This can also be used with a pipe function. + +```liquid +{{ 10 | divide 2 }} // 5 +``` + +Please take careful note of the order or arguments. + +##### `modulo` + +Returns the modulo of the second value from the first. + +```liquid +{{ modulo 2 5 }} // 1 +``` + +This can also be used with a pipe function. + +```liquid +{{ 5 | modulo 2 }} // 1 +``` + +Please take careful note of the order of arguments. + +## Plugins + +### Authoring Plugins + +For some use cases, it may be necessary to write a plugin that offloads work to +another system. This is especially useful for things that may not fit in the +"standard library" of Consul Template, but still need to be shared across +multiple instances. + +Consul Template plugins must have the following API: + +```shell +$ NAME [INPUT...] +``` + +- `NAME` - the name of the plugin - this is also the name of the binary, either + a full path or just the program name. It will be executed in a shell with the + inherited `PATH` so e.g. the plugin `cat` will run the first executable `cat` + that is found on the `PATH`. + +- `INPUT` - input from the template - this will always be JSON if provided + +#### Important Notes + +- Plugins execute user-provided scripts and pass in potentially sensitive data + from Consul or Vault. Nothing is validated or protected by Consul Template, + so all necessary precautions and considerations should be made by template + authors + +- Plugin output must be returned as a string on stdout. Only stdout will be + parsed for output. Be sure to log all errors, debugging messages onto stderr + to avoid errors when Consul Template returns the value. + +- Always `exit 0` or Consul Template will assume the plugin failed to execute + +- Ensure the empty input case is handled correctly (see [Multi-phase execution](#multi-phase-execution)) + +- Data piped into the plugin is appended after any parameters given explicitly (eg `{{ "sample-data" | plugin "my-plugin" "some-parameter"}}` will call `my-plugin some-parameter sample-data`) + +Here is a sample plugin in a few different languages that removes any JSON keys +that start with an underscore and returns the JSON string: + +```ruby +#! /usr/bin/env ruby +require "json" + +if ARGV.empty? + puts JSON.fast_generate({}) + Kernel.exit(0) +end + +hash = JSON.parse(ARGV.first) +hash.reject! { |k, _| k.start_with?("_") } +puts JSON.fast_generate(hash) +Kernel.exit(0) +``` + +```go +func main() { + arg := []byte(os.Args[1]) + + var parsed map[string]interface{} + if err := json.Unmarshal(arg, &parsed); err != nil { + fmt.Fprintln(os.Stderr, fmt.Sprintf("err: %s", err)) + os.Exit(1) + } + + for k, _ := range parsed { + if string(k[0]) == "_" { + delete(parsed, k) + } + } + + result, err := json.Marshal(parsed) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Sprintf("err: %s", err)) + os.Exit(1) + } + + fmt.Fprintln(os.Stdout, fmt.Sprintf("%s", result)) + os.Exit(0) +} +``` + +## Caveats + +### Once Mode + +In Once mode, Consul Template will wait for all dependencies to be rendered. If +a template specifies a dependency (a request) that does not exist in Consul, +once mode will wait until Consul returns data for that dependency. Please note +that "returned data" and "empty data" are not mutually exclusive. + +When you query for all healthy services named "foo" (`{{ service "foo" }}`), you +are asking Consul - "give me all the healthy services named foo". If there are +no services named foo, the response is the empty array. This is also the same +response if there are no _healthy_ services named foo. + +Consul template processes input templates multiple times, since the first result +could impact later dependencies: + +```liquid +{{ range services }} +{{ range service .Name }} +{{ end }} +{{ end }} +``` + +In this example, we have to process the output of `services` before we can +lookup each `service`, since the inner loops cannot be evaluated until the outer +loop returns a response. Consul Template waits until it gets a response from +Consul for all dependencies before rendering a template. It does not wait until +that response is non-empty though. + +### Exec Mode + +As of version 0.16.0, Consul Template has the ability to maintain an arbitrary +child process (similar to [envconsul](https://github.com/hashicorp/envconsul)). +This mode is most beneficial when running Consul Template in a container or on a +scheduler like [Nomad](https://www.nomadproject.io) or Kubernetes. When +activated, Consul Template will spawn and manage the lifecycle of the child +process. + +This mode is best-explained through example. Consider a simple application that +reads a configuration file from disk and spawns a server from that +configuration. + +```sh +$ consul-template \ + -template "/tmp/config.ctmpl:/tmp/server.conf" \ + -exec "/bin/my-server -config /tmp/server.conf" +``` + +When Consul Template starts, it will pull the required dependencies and populate +the `/tmp/server.conf`, which the `my-server` binary consumes. After that +template is rendered completely the first time, Consul Template spawns and +manages a child process. When any of the list templates change, Consul Template +will send a configurable reload signal to the child process. Additionally, +Consul Template will proxy any signals it receives to the child process. This +enables a scheduler to control the lifecycle of the process and also eases the +friction of running inside a container. + +A common point of confusion is that the command string behaves the same as the +shell; it does not. In the shell, when you run `foo | bar` or `foo > bar`, that +is actually running as a subprocess of your shell (bash, zsh, csh, etc.). When +Consul Template spawns the exec process, it runs outside of your shell. This +behavior is _different_ from when Consul Template executes the template-specific +reload command. If you want the ability to pipe or redirect in the exec command, +you will need to spawn the process in subshell, for example: + +```hcl +exec { + command = "/bin/bash -c 'my-server > /var/log/my-server.log'" +} +``` + +Note that when spawning like this, most shells do not proxy signals to their +child by default, so your child process will not receive the signals that Consul +Template sends to the shell. You can avoid this by writing a tiny shell wrapper +and executing that instead: + +```bash +#!/usr/bin/env bash +trap "kill -TERM $child" SIGTERM + +/bin/my-server -config /tmp/server.conf +child=$! +wait "$child" +``` + +Alternatively, you can use your shell's exec function directly, if it exists: + +```bash +#!/usr/bin/env bash +exec /bin/my-server -config /tmp/server.conf > /var/log/my-server.log +``` + +There are some additional caveats with Exec Mode, which should be considered +carefully before use: + +- If the child process dies, the Consul Template process will also die. Consul + Template **does not supervise the process!** This is generally the + responsibility of the scheduler or init system. + +- The child process must remain in the foreground. This is a requirement for + Consul Template to manage the process and send signals. + +- The exec command will only start after _all_ templates have been rendered at + least once. One may have multiple templates for a single Consul Template + process, all of which must be rendered before the process starts. Consider + something like an nginx or apache configuration where both the process + configuration file and individual site configuration must be written in order + for the service to successfully start. + +- After the child process is started, any change to any dependent template will + cause the reload signal to be sent to the child process. If no reload signal + is provided, Consul Template will kill the process and spawn a new instance. + The reload signal can be specified and customized via the CLI or configuration + file. + +- When Consul Template is stopped gracefully, it will send the configurable kill + signal to the child process. The default value is SIGTERM, but it can be + customized via the CLI or configuration file. + +- Consul Template will forward all signals it receives to the child process + **except** its defined `reload_signal` and `kill_signal`. If you disable these + signals, Consul Template will forward them to the child process. + +- It is not possible to have more than one exec command (although each template + can still have its own reload command). + +- Individual template reload commands still fire independently of the exec + command. + +### De-Duplication Mode + +Consul Template works by parsing templates to determine what data is needed and +then watching Consul for any changes to that data. This allows Consul Template +to efficiently re-render templates when a change occurs. However, if there are +many instances of Consul Template rendering a common template there is a linear +duplication of work as each instance is querying the same data. + +To make this pattern more efficient Consul Template supports work de-duplication +across instances. This can be enabled with the `-dedup` flag or via the +`deduplicate` configuration block. Once enabled, Consul Template uses leader +election on a per-template basis to have only a single node perform the queries. +Results are shared among other instances rendering the same template by passing +compressed data through the Consul K/V store. + +Please note that no Vault data will be stored in the compressed template. +Because ACLs around Vault are typically more closely controlled than those ACLs +around Consul's KV, Consul Template will still request the secret from Vault on +each iteration. + +When running in de-duplication mode, it is important that local template +functions resolve correctly. For example, you may have a local template function +that relies on the `env` helper like this: + +```hcl +{{ key (env "KEY") }} +``` + +It is crucial that the environment variable `KEY` in this example is consistent +across all machines engaged in de-duplicating this template. If the values are +different, Consul Template will be unable to resolve the template, and you will +not get a successful render. + +### Termination on Error + +By default Consul Template is highly fault-tolerant. If Consul is unreachable or +a template changes, Consul Template will happily continue running. The only +exception to this rule is if the optional `command` exits non-zero. In this +case, Consul Template will also exit non-zero. The reason for this decision is +so the user can easily configure something like Upstart or God to manage Consul +Template as a service. + +If you want Consul Template to continue watching for changes, even if the +optional command argument fails, you can append `|| true` to your command. Note +that `||` is a "shell-ism", not a built-in function. You will also need to run +your command under a shell: + +```shell +$ consul-template \ + -template "in.ctmpl:out.file:/bin/bash -c 'service nginx restart || true'" +``` + +In this example, even if the Nginx restart command returns non-zero, the overall +function will still return an OK exit code; Consul Template will continue to run +as a service. Additionally, if you have complex logic for restarting your +service, you can intelligently choose when you want Consul Template to exit and +when you want it to continue to watch for changes. For these types of complex +scripts, we recommend using a custom sh or bash script instead of putting the +logic directly in the `consul-template` command or configuration file. + +### Command Environment + +The current processes environment is used when executing commands with the following additional environment variables: + +- `CONSUL_HTTP_ADDR` +- `CONSUL_HTTP_TOKEN` +- `CONSUL_HTTP_AUTH` +- `CONSUL_HTTP_SSL` +- `CONSUL_HTTP_SSL_VERIFY` + +These environment variables are exported with their current values when the +command executes. Other Consul tooling reads these environment variables, +providing smooth integration with other Consul tools (like `consul maint` or +`consul lock`). Additionally, exposing these environment variables gives power +users the ability to further customize their command script. + +### Multi-phase Execution + +Consul Template does an n-pass evaluation of templates, accumulating +dependencies on each pass. This is required due to nested dependencies, such as: + +```liquid +{{ range services }} +{{ range service .Name }} + {{ .Address }} +{{ end }}{{ end }} +``` + +During the first pass, Consul Template does not know any of the services in +Consul, so it has to perform a query. When those results are returned, the +inner-loop is then evaluated with that result, potentially creating more queries +and watches. + +Because of this implementation, template functions need a default value that is +an acceptable parameter to a `range` function (or similar), but does not +actually execute the inner loop (which would cause a panic). This is important +to mention because complex templates **must** account for the "empty" case. For +example, the following **will not work**: + +```liquid +{{ with index (service "foo") 0 }} +# ... +{{ end }} +``` + +This will raise an error like: + +```text +: error calling index: index out of range: 0 +``` + +That is because, during the _first_ evaluation of the template, the `service` +key is returning an empty slice. You can account for this in your template like +so: + +```liquid +{{ with service "foo" }} +{{ with index . 0 }} +{{ .Node }}{{ end }}{{ end }} +``` + +This will still add the dependency to the list of watches, but will not +evaluate the inner-if, avoiding the out-of-index error. + +## Running and Process Lifecycle + +While there are multiple ways to run Consul Template, the most common pattern is +to run Consul Template as a system service. When Consul Template first starts, +it reads any configuration files and templates from disk and loads them into +memory. From that point forward, changes to the files on disk do not propagate +to running process without a reload. + +The reason for this behavior is simple and aligns with other tools like haproxy. +A user may want to perform pre-flight validation checks on the configuration or +templates before loading them into the process. Additionally, a user may want to +update configuration and templates simultaneously. Having Consul Template +automatically watch and reload those files on changes is both operationally +dangerous and against some of the paradigms of modern infrastructure. Instead, +Consul Template listens for the `SIGHUP` syscall to trigger a configuration +reload. If you update configuration or templates, simply send `HUP` to the +running Consul Template process and Consul Template will reload all the +configurations and templates from disk. + +## Debugging + +Consul Template can print verbose debugging output. To set the log level for +Consul Template, use the `-log-level` flag: + +```shell +$ consul-template -log-level info ... +``` + +```text + [INFO] (cli) received redis from Watcher + [INFO] (cli) invoking Runner +# ... +``` + +You can also specify the level as debug: + +```shell +$ consul-template -log-level debug ... +``` + +```text + [DEBUG] (cli) creating Runner + [DEBUG] (cli) creating Consul API client + [DEBUG] (cli) creating Watcher + [DEBUG] (cli) looping for data + [DEBUG] (watcher) starting watch + [DEBUG] (watcher) all pollers have started, waiting for finish + [DEBUG] (redis) starting poll + [DEBUG] (service redis) querying Consul with &{...} + [DEBUG] (service redis) Consul returned 2 services + [DEBUG] (redis) writing data to channel + [DEBUG] (redis) starting poll + [INFO] (cli) received redis from Watcher + [INFO] (cli) invoking Runner + [DEBUG] (service redis) querying Consul with &{...} +# ... +``` + + +## FAQ + +**Q: How is this different than confd?**
+A: The answer is simple: Service Discovery as a first class citizen. You are also encouraged to read [this Pull Request](https://github.com/kelseyhightower/confd/pull/102) on the project for more background information. We think confd is a great project, but Consul Template fills a missing gap. Additionally, Consul Template has first class integration with [Vault](https://vaultproject.io), making it easy to incorporate secret material like database credentials or API tokens into configuration files. + +**Q: How is this different than Puppet/Chef/Ansible/Salt?**
+A: Configuration management tools are designed to be used in unison with Consul Template. Instead of rendering a stale configuration file, use your configuration management software to render a dynamic template that will be populated by [Consul][]. + + +## Contributing + +To build and install Consul Template locally, you will need to install the +Docker engine: + +- [Docker for Mac](https://docs.docker.com/engine/installation/mac/) +- [Docker for Windows](https://docs.docker.com/engine/installation/windows/) +- [Docker for Linux](https://docs.docker.com/engine/installation/linux/ubuntulinux/) + +Clone the repository: + +```shell +$ git clone https://github.com/hashicorp/consul-template.git +``` + +To compile the `consul-template` binary for your local machine: + +```shell +$ make dev +``` + +This will compile the `consul-template` binary into `bin/consul-template` as +well as your `$GOPATH` and run the test suite. + +If you want to compile a specific binary, set `XC_OS` and `XC_ARCH` or run the +following to generate all binaries: + +```shell +$ make bin +``` + +If you want to run the tests, first [install consul locally](https://www.consul.io/docs/install/index.html), then: + +```shell +$ make test +``` + +Or to run a specific test in the suite: + +```shell +go test ./... -run SomeTestFunction_name +``` + +[consul]: https://www.consul.io "Consul by HashiCorp" +[examples]: (https://github.com/hashicorp/consul-template/tree/master/examples) "Consul Template Examples" +[hcl]: https://github.com/hashicorp/hcl "HashiCorp Configuration Language (hcl)" +[releases]: https://releases.hashicorp.com/consul-template "Consul Template Releases" +[text-template]: https://golang.org/pkg/text/template/ "Go's text/template package" +[vault]: https://www.vaultproject.io "Vault by HashiCorp" diff --git a/vendor/github.com/hashicorp/consul-template/cli.go b/vendor/github.com/hashicorp/consul-template/cli.go index cde774cf488..1ac7483d06d 100644 --- a/vendor/github.com/hashicorp/consul-template/cli.go +++ b/vendor/github.com/hashicorp/consul-template/cli.go @@ -193,6 +193,14 @@ func (cli *CLI) ParseFlags(args []string) (*config.Config, []string, bool, bool, c := config.DefaultConfig() + if s := os.Getenv("CT_LOCAL_CONFIG"); s != "" { + envConfig, err := config.Parse(s) + if err != nil { + return nil, nil, false, false, false, err + } + c = c.Merge(envConfig) + } + // configPaths stores the list of configuration paths on disk configPaths := make([]string, 0, 6) @@ -505,6 +513,11 @@ func (cli *CLI) ParseFlags(args []string) (*config.Config, []string, bool, bool, return nil }), "vault-token", "") + flags.Var((funcVar)(func(s string) error { + c.Vault.VaultAgentTokenFile = config.String(s) + return nil + }), "vault-agent-token-file", "") + flags.Var((funcBoolVar)(func(b bool) error { c.Vault.UnwrapToken = config.Bool(b) return nil @@ -760,6 +773,9 @@ Options: -vault-token= Sets the Vault API token + -vault-agent-token-file= + File to read Vault API token from. + -vault-transport-dial-keep-alive= Sets the amount of time to use for keep-alives diff --git a/vendor/github.com/hashicorp/consul-template/config/vault.go b/vendor/github.com/hashicorp/consul-template/config/vault.go index c5c1b32f2cd..e25e776154e 100644 --- a/vendor/github.com/hashicorp/consul-template/config/vault.go +++ b/vendor/github.com/hashicorp/consul-template/config/vault.go @@ -42,6 +42,10 @@ type VaultConfig struct { // new secret to be read. Grace *time.Duration `mapstructure:"grace"` + // Namespace is the Vault namespace to use for reading/writing secrets. This can + // also be set via the VAULT_NAMESPACE environment variable. + Namespace *string `mapstructure:"namespace"` + // RenewToken renews the Vault token. RenewToken *bool `mapstructure:"renew_token"` @@ -53,9 +57,16 @@ type VaultConfig struct { // Token is the Vault token to communicate with for requests. It may be // a wrapped token or a real token. This can also be set via the VAULT_TOKEN - // environment variable. + // environment variable, or via the VaultAgentTokenFile. Token *string `mapstructure:"token" json:"-"` + // VaultAgentTokenFile is the path of file that contains a Vault Agent token. + // If vault_agent_token_file is specified: + // - Consul Template will not try to renew the Vault token. + // - Consul Template will periodically stat the file and update the token if it has + // changed. + VaultAgentTokenFile *string `mapstructure:"vault_agent_token_file" json:"-"` + // Transport configures the low-level network connection details. Transport *TransportConfig `mapstructure:"transport"` @@ -91,6 +102,8 @@ func (c *VaultConfig) Copy() *VaultConfig { o.Grace = c.Grace + o.Namespace = c.Namespace + o.RenewToken = c.RenewToken if c.Retry != nil { @@ -103,6 +116,8 @@ func (c *VaultConfig) Copy() *VaultConfig { o.Token = c.Token + o.VaultAgentTokenFile = c.VaultAgentTokenFile + if c.Transport != nil { o.Transport = c.Transport.Copy() } @@ -142,6 +157,10 @@ func (c *VaultConfig) Merge(o *VaultConfig) *VaultConfig { r.Grace = o.Grace } + if o.Namespace != nil { + r.Namespace = o.Namespace + } + if o.RenewToken != nil { r.RenewToken = o.RenewToken } @@ -158,6 +177,10 @@ func (c *VaultConfig) Merge(o *VaultConfig) *VaultConfig { r.Token = o.Token } + if o.VaultAgentTokenFile != nil { + r.VaultAgentTokenFile = o.VaultAgentTokenFile + } + if o.Transport != nil { r.Transport = r.Transport.Merge(o.Transport) } @@ -181,6 +204,10 @@ func (c *VaultConfig) Finalize() { c.Grace = TimeDuration(DefaultVaultGrace) } + if c.Namespace == nil { + c.Namespace = stringFromEnv([]string{"VAULT_NAMESPACE"}, "") + } + if c.RenewToken == nil { c.RenewToken = boolFromEnv([]string{ "VAULT_RENEW_TOKEN", @@ -219,18 +246,19 @@ func (c *VaultConfig) Finalize() { } c.SSL.Finalize() + // Order of precedence + // 1. `vault_agent_token_file` configuration value + // 2. `token` configuration value` + // 3. `VAULT_TOKEN` environment variable if c.Token == nil { c.Token = stringFromEnv([]string{ "VAULT_TOKEN", }, "") + } - if StringVal(c.Token) == "" { - if homePath != "" { - c.Token = stringFromFile([]string{ - homePath + "/.vault-token", - }, "") - } - } + if c.VaultAgentTokenFile != nil { + c.Token = stringFromFile([]string{*c.VaultAgentTokenFile}, "") + c.RenewToken = Bool(false) } if c.Transport == nil { @@ -259,20 +287,24 @@ func (c *VaultConfig) GoString() string { "Address:%s, "+ "Enabled:%s, "+ "Grace:%s, "+ + "Namespace:%s,"+ "RenewToken:%s, "+ "Retry:%#v, "+ "SSL:%#v, "+ "Token:%t, "+ + "VaultAgentTokenFile:%t, "+ "Transport:%#v, "+ "UnwrapToken:%s"+ "}", StringGoString(c.Address), - TimeDurationGoString(c.Grace), BoolGoString(c.Enabled), + TimeDurationGoString(c.Grace), + StringGoString(c.Namespace), BoolGoString(c.RenewToken), c.Retry, c.SSL, StringPresent(c.Token), + StringPresent(c.VaultAgentTokenFile), c.Transport, BoolGoString(c.UnwrapToken), ) diff --git a/vendor/github.com/hashicorp/consul-template/dependency/catalog_node.go b/vendor/github.com/hashicorp/consul-template/dependency/catalog_node.go index d8fb1f6655e..12ef7633dd0 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/catalog_node.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/catalog_node.go @@ -43,6 +43,7 @@ type CatalogNodeService struct { ID string Service string Tags ServiceTags + Meta map[string]string Port int Address string EnableTagOverride bool @@ -116,6 +117,7 @@ func (d *CatalogNodeQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interf ID: v.ID, Service: v.Service, Tags: ServiceTags(deepCopyAndSortTags(v.Tags)), + Meta: v.Meta, Port: v.Port, Address: v.Address, EnableTagOverride: v.EnableTagOverride, diff --git a/vendor/github.com/hashicorp/consul-template/dependency/catalog_service.go b/vendor/github.com/hashicorp/consul-template/dependency/catalog_service.go index a56516915d3..8b94a59961d 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/catalog_service.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/catalog_service.go @@ -34,6 +34,7 @@ type CatalogService struct { ServiceName string ServiceAddress string ServiceTags ServiceTags + ServiceMeta map[string]string ServicePort int } @@ -109,6 +110,7 @@ func (d *CatalogServiceQuery) Fetch(clients *ClientSet, opts *QueryOptions) (int ServiceName: s.ServiceName, ServiceAddress: s.ServiceAddress, ServiceTags: ServiceTags(deepCopyAndSortTags(s.ServiceTags)), + ServiceMeta: s.ServiceMeta, ServicePort: s.ServicePort, }) } diff --git a/vendor/github.com/hashicorp/consul-template/dependency/client_set.go b/vendor/github.com/hashicorp/consul-template/dependency/client_set.go index c6cbbb7e961..e2bceb77382 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/client_set.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/client_set.go @@ -62,6 +62,7 @@ type CreateConsulClientInput struct { // CreateVaultClientInput is used as input to the CreateVaultClient function. type CreateVaultClientInput struct { Address string + Namespace string Token string UnwrapToken bool SSLEnabled bool @@ -265,6 +266,11 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error { return fmt.Errorf("client set: vault: %s", err) } + // Set the namespace if given. + if i.Namespace != "" { + client.SetNamespace(i.Namespace) + } + // Set the token if given if i.Token != "" { client.SetToken(i.Token) diff --git a/vendor/github.com/hashicorp/consul-template/dependency/health_service.go b/vendor/github.com/hashicorp/consul-template/dependency/health_service.go index 50112d7d7ad..215f53d0d2a 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/health_service.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/health_service.go @@ -43,6 +43,7 @@ type HealthService struct { NodeAddress string NodeTaggedAddresses map[string]string NodeMeta map[string]string + ServiceMeta map[string]string Address string ID string Name string @@ -162,6 +163,7 @@ func (d *HealthServiceQuery) Fetch(clients *ClientSet, opts *QueryOptions) (inte NodeAddress: entry.Node.Address, NodeTaggedAddresses: entry.Node.TaggedAddresses, NodeMeta: entry.Node.Meta, + ServiceMeta: entry.Service.Meta, Address: address, ID: entry.Service.ID, Name: entry.Service.Service, diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_agent_token.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_agent_token.go new file mode 100644 index 00000000000..1ce339ea6e2 --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_agent_token.go @@ -0,0 +1,121 @@ +package dependency + +import ( + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/pkg/errors" +) + +var ( + // Ensure implements + _ Dependency = (*VaultAgentTokenQuery)(nil) +) + +const ( + // VaultAgentTokenSleepTime is the amount of time to sleep between queries, since + // the fsnotify library is not compatible with solaris and other OSes yet. + VaultAgentTokenSleepTime = 15 * time.Second +) + +// VaultAgentTokenQuery is the dependency to Vault Agent token +type VaultAgentTokenQuery struct { + stopCh chan struct{} + path string + stat os.FileInfo +} + +// NewVaultAgentTokenQuery creates a new dependency. +func NewVaultAgentTokenQuery(path string) (*VaultAgentTokenQuery, error) { + return &VaultAgentTokenQuery{ + stopCh: make(chan struct{}, 1), + path: path, + }, nil +} + +// Fetch retrieves this dependency and returns the result or any errors that +// occur in the process. +func (d *VaultAgentTokenQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { + log.Printf("[TRACE] %s: READ %s", d, d.path) + + select { + case <-d.stopCh: + log.Printf("[TRACE] %s: stopped", d) + return "", nil, ErrStopped + case r := <-d.watch(d.stat): + if r.err != nil { + return "", nil, errors.Wrap(r.err, d.String()) + } + + log.Printf("[TRACE] %s: reported change", d) + + token, err := ioutil.ReadFile(d.path) + if err != nil { + return "", nil, errors.Wrap(err, d.String()) + } + + d.stat = r.stat + clients.Vault().SetToken(strings.TrimSpace(string(token))) + } + + return respWithMetadata("") +} + +// CanShare returns if this dependency is sharable. +func (d *VaultAgentTokenQuery) CanShare() bool { + return false +} + +// Stop halts the dependency's fetch function. +func (d *VaultAgentTokenQuery) Stop() { + close(d.stopCh) +} + +// String returns the human-friendly version of this dependency. +func (d *VaultAgentTokenQuery) String() string { + return "vault-agent.token" +} + +// Type returns the type of this dependency. +func (d *VaultAgentTokenQuery) Type() Type { + return TypeVault +} + +// watch watches the file for changes +func (d *VaultAgentTokenQuery) watch(lastStat os.FileInfo) <-chan *watchResult { + ch := make(chan *watchResult, 1) + + go func(lastStat os.FileInfo) { + for { + stat, err := os.Stat(d.path) + if err != nil { + select { + case <-d.stopCh: + return + case ch <- &watchResult{err: err}: + return + } + } + + changed := lastStat == nil || + lastStat.Size() != stat.Size() || + lastStat.ModTime() != stat.ModTime() + + if changed { + select { + case <-d.stopCh: + return + case ch <- &watchResult{stat: stat}: + return + } + } + + time.Sleep(VaultAgentTokenSleepTime) + } + }(lastStat) + + return ch +} diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go index b554588dfb5..42b331fd0e3 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go @@ -3,6 +3,8 @@ package dependency import ( "log" "math/rand" + "path" + "strings" "time" "github.com/hashicorp/vault/api" @@ -79,12 +81,19 @@ func vaultRenewDuration(s *Secret) time.Duration { // Convert to float seconds. sleep := float64(time.Duration(base) * time.Second) - // Renew at 1/3 the remaining lease. This will give us an opportunity to retry - // at least one more time should the first renewal fail. - sleep = sleep / 3.0 - - // Use a randomness so many clients do not hit Vault simultaneously. - sleep = sleep * (rand.Float64() + 1) / 2.0 + if vaultSecretRenewable(s) { + // Renew at 1/3 the remaining lease. This will give us an opportunity to retry + // at least one more time should the first renewal fail. + sleep = sleep / 3.0 + + // Use some randomness so many clients do not hit Vault simultaneously. + sleep = sleep * (rand.Float64() + 1) / 2.0 + } else { + // For non-renewable leases set the renew duration to use much of the secret + // lease as possible. Use a stagger over 85%-95% of the lease duration so that + // many clients do not hit Vault simultaneously. + sleep = sleep * (.85 + rand.Float64()*0.1) + } return time.Duration(sleep) } @@ -193,3 +202,73 @@ func updateSecret(ours *Secret, theirs *api.Secret) { } } } + +func isKVv2(client *api.Client, path string) (string, bool, error) { + // We don't want to use a wrapping call here so save any custom value and + // restore after + currentWrappingLookupFunc := client.CurrentWrappingLookupFunc() + client.SetWrappingLookupFunc(nil) + defer client.SetWrappingLookupFunc(currentWrappingLookupFunc) + currentOutputCurlString := client.OutputCurlString() + client.SetOutputCurlString(false) + defer client.SetOutputCurlString(currentOutputCurlString) + + r := client.NewRequest("GET", "/v1/sys/internal/ui/mounts/"+path) + resp, err := client.RawRequest(r) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + // If we get a 404 we are using an older version of vault, default to + // version 1 + if resp != nil && resp.StatusCode == 404 { + return "", false, nil + } + + return "", false, err + } + + secret, err := api.ParseSecret(resp.Body) + if err != nil { + return "", false, err + } + var mountPath string + if mountPathRaw, ok := secret.Data["path"]; ok { + mountPath = mountPathRaw.(string) + } + var mountType string + if mountTypeRaw, ok := secret.Data["type"]; ok { + mountType = mountTypeRaw.(string) + } + options := secret.Data["options"] + if options == nil { + return mountPath, false, nil + } + versionRaw := options.(map[string]interface{})["version"] + if versionRaw == nil { + return mountPath, false, nil + } + version := versionRaw.(string) + switch version { + case "", "1": + return mountPath, false, nil + case "2": + return mountPath, mountType == "kv", nil + } + + return mountPath, false, nil +} + +func addPrefixToVKVPath(p, mountPath, apiPrefix string) string { + switch { + case p == mountPath, p == strings.TrimSuffix(mountPath, "/"): + return path.Join(mountPath, apiPrefix) + default: + p = strings.TrimPrefix(p, mountPath) + // Don't add /data to the path if it's been added manually. + if strings.HasPrefix(p, apiPrefix) { + return path.Join(mountPath, p) + } + return path.Join(mountPath, apiPrefix, p) + } +} diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go index 3d86fe1c324..4f0e21b7211 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go @@ -20,8 +20,11 @@ var ( type VaultReadQuery struct { stopCh chan struct{} - path string - secret *Secret + rawPath string + queryValues url.Values + secret *Secret + isKVv2 *bool + secretPath string // vaultSecret is the actual Vault secret which we are renewing vaultSecret *api.Secret @@ -35,9 +38,15 @@ func NewVaultReadQuery(s string) (*VaultReadQuery, error) { return nil, fmt.Errorf("vault.read: invalid format: %q", s) } + secretURL, err := url.Parse(s) + if err != nil { + return nil, err + } + return &VaultReadQuery{ - stopCh: make(chan struct{}, 1), - path: s, + stopCh: make(chan struct{}, 1), + rawPath: secretURL.Path, + queryValues: secretURL.Query(), }, nil } @@ -123,7 +132,7 @@ func (d *VaultReadQuery) Stop() { // String returns the human-friendly version of this dependency. func (d *VaultReadQuery) String() string { - return fmt.Sprintf("vault.read(%s)", d.path) + return fmt.Sprintf("vault.read(%s)", d.rawPath) } // Type returns the type of this dependency. @@ -132,16 +141,34 @@ func (d *VaultReadQuery) Type() Type { } func (d *VaultReadQuery) readSecret(clients *ClientSet, opts *QueryOptions) (*api.Secret, error) { + vaultClient := clients.Vault() + + // Check whether this secret refers to a KV v2 entry if we haven't yet. + if d.isKVv2 == nil { + mountPath, isKVv2, err := isKVv2(vaultClient, d.rawPath) + if err != nil { + return nil, errors.Wrap(err, d.String()) + } + + if isKVv2 { + d.secretPath = addPrefixToVKVPath(d.rawPath, mountPath, "data") + } else { + d.secretPath = d.rawPath + } + d.isKVv2 = &isKVv2 + } + + queryString := d.queryValues.Encode() log.Printf("[TRACE] %s: GET %s", d, &url.URL{ - Path: "/v1/" + d.path, - RawQuery: opts.String(), + Path: "/v1/" + d.secretPath, + RawQuery: queryString, }) - vaultSecret, err := clients.Vault().Logical().Read(d.path) + vaultSecret, err := vaultClient.Logical().ReadWithData(d.secretPath, d.queryValues) if err != nil { return nil, errors.Wrap(err, d.String()) } if vaultSecret == nil { - return nil, fmt.Errorf("no secret exists at %s", d.path) + return nil, fmt.Errorf("no secret exists at %s", d.secretPath) } return vaultSecret, nil } diff --git a/vendor/github.com/hashicorp/consul-template/manager/runner.go b/vendor/github.com/hashicorp/consul-template/manager/runner.go index ebe3e801aa2..13da0e83fa7 100644 --- a/vendor/github.com/hashicorp/consul-template/manager/runner.go +++ b/vendor/github.com/hashicorp/consul-template/manager/runner.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/consul-template/child" "github.com/hashicorp/consul-template/config" dep "github.com/hashicorp/consul-template/dependency" + "github.com/hashicorp/consul-template/renderer" "github.com/hashicorp/consul-template/template" "github.com/hashicorp/consul-template/watch" "github.com/hashicorp/go-multierror" @@ -161,6 +162,12 @@ type RenderEvent struct { // LastDidRender marks the last time the template was written to disk. LastDidRender time.Time + + // ForQuiescence determines if this event is returned early in the + // render loop due to quiescence. When evaluating if all templates have + // been rendered we need to know if the event is triggered by quiesence + // and if we can skip evaluating it as a render event for those purposes + ForQuiescence bool } // NewRunner accepts a slice of TemplateConfigs and returns a pointer to the new @@ -397,7 +404,7 @@ func (r *Runner) Stop() { r.stopChild() if err := r.deletePid(); err != nil { - log.Printf("[WARN] (runner) could not remove pid at %q: %s", + log.Printf("[WARN] (runner) could not remove pid at %v: %s", r.config.PidFile, err) } @@ -730,6 +737,8 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (* // We do not want to render the templates yet. if q, ok := r.quiescenceMap[tmpl.ID()]; ok { q.tick() + // This event is being returned early for quiescence + event.ForQuiescence = true return event, nil } @@ -739,7 +748,7 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (* log.Printf("[DEBUG] (runner) rendering %s", templateConfig.Display()) // Render the template, taking dry mode into account - result, err := Render(&RenderInput{ + result, err := renderer.Render(&renderer.RenderInput{ Backup: config.BoolVal(templateConfig.Backup), Contents: result.Output, CreateDestDirs: config.BoolVal(templateConfig.CreateDestDirs), @@ -957,6 +966,13 @@ func (r *Runner) allTemplatesRendered() bool { return false } + // Skip evaluation of events from quiescence as they will + // be default unrendered as we are still waiting for the + // specified period + if event.ForQuiescence { + continue + } + // The template might already exist on disk with the exact contents, but // we still want to count that as "rendered" [GH-1000]. if !event.DidRender && !event.WouldRender { @@ -1222,6 +1238,7 @@ func newClientSet(c *config.Config) (*dep.ClientSet, error) { if err := clients.CreateVaultClient(&dep.CreateVaultClientInput{ Address: config.StringVal(c.Vault.Address), + Namespace: config.StringVal(c.Vault.Namespace), Token: config.StringVal(c.Vault.Token), UnwrapToken: config.BoolVal(c.Vault.UnwrapToken), SSLEnabled: config.BoolVal(c.Vault.SSL.Enabled), @@ -1250,11 +1267,12 @@ func newWatcher(c *config.Config, clients *dep.ClientSet, once bool) (*watch.Wat log.Printf("[INFO] (runner) creating watcher") w, err := watch.NewWatcher(&watch.NewWatcherInput{ - Clients: clients, - MaxStale: config.TimeDurationVal(c.MaxStale), - Once: once, - RenewVault: clients.Vault().Token() != "" && config.BoolVal(c.Vault.RenewToken), - RetryFuncConsul: watch.RetryFunc(c.Consul.Retry.RetryFunc()), + Clients: clients, + MaxStale: config.TimeDurationVal(c.MaxStale), + Once: once, + RenewVault: clients.Vault().Token() != "" && config.BoolVal(c.Vault.RenewToken), + VaultAgentTokenFile: config.StringVal(c.Vault.VaultAgentTokenFile), + RetryFuncConsul: watch.RetryFunc(c.Consul.Retry.RetryFunc()), // TODO: Add a sane default retry - right now this only affects "local" // dependencies like reading a file from disk. RetryFuncDefault: nil, diff --git a/vendor/github.com/hashicorp/consul-template/renderer/file_perms.go b/vendor/github.com/hashicorp/consul-template/renderer/file_perms.go new file mode 100644 index 00000000000..d89b2f02c6e --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/renderer/file_perms.go @@ -0,0 +1,22 @@ +//+build !windows + +package renderer + +import ( + "os" + "syscall" +) + +func preserveFilePermissions(path string, fileInfo os.FileInfo) error { + sysInfo := fileInfo.Sys() + if sysInfo != nil { + stat, ok := sysInfo.(*syscall.Stat_t) + if ok { + if err := os.Chown(path, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + } + } + + return nil +} diff --git a/vendor/github.com/hashicorp/consul-template/renderer/file_perms_windows.go b/vendor/github.com/hashicorp/consul-template/renderer/file_perms_windows.go new file mode 100644 index 00000000000..cae35cf51ac --- /dev/null +++ b/vendor/github.com/hashicorp/consul-template/renderer/file_perms_windows.go @@ -0,0 +1,9 @@ +//+build windows + +package renderer + +import "os" + +func preserveFilePermissions(path string, fileInfo os.FileInfo) error { + return nil +} diff --git a/vendor/github.com/hashicorp/consul-template/manager/renderer.go b/vendor/github.com/hashicorp/consul-template/renderer/renderer.go similarity index 95% rename from vendor/github.com/hashicorp/consul-template/manager/renderer.go rename to vendor/github.com/hashicorp/consul-template/renderer/renderer.go index 799826470b4..34e68a2b15c 100644 --- a/vendor/github.com/hashicorp/consul-template/manager/renderer.go +++ b/vendor/github.com/hashicorp/consul-template/renderer/renderer.go @@ -1,10 +1,11 @@ -package manager +package renderer import ( "bytes" "fmt" "io" "io/ioutil" + "log" "os" "path/filepath" @@ -148,6 +149,12 @@ func AtomicWrite(path string, createDestDirs bool, contents []byte, perms os.Fil } } else { perms = currentInfo.Mode() + + // The file exists, so try to preserve the ownership as well. + if err := preserveFilePermissions(f.Name(), currentInfo); err != nil { + log.Printf("[WARN] (runner) could not preserve file permissions for %q: %v", + f.Name(), err) + } } } diff --git a/vendor/github.com/hashicorp/consul-template/template/funcs.go b/vendor/github.com/hashicorp/consul-template/template/funcs.go index f354b0d452d..c0048df016c 100644 --- a/vendor/github.com/hashicorp/consul-template/template/funcs.go +++ b/vendor/github.com/hashicorp/consul-template/template/funcs.go @@ -657,6 +657,9 @@ func in(l, v interface{}) (bool, error) { // Indent prefixes each line of a string with the specified number of spaces func indent(spaces int, s string) (string, error) { + if spaces < 0 { + return "", fmt.Errorf("indent value must be a positive integer") + } var output, prefix []byte var sp bool var size int @@ -669,7 +672,7 @@ func indent(spaces int, s string) (string, error) { } output = append(output, c) sp = c == '\n' - size += 1 + size++ } return string(output[:size]), nil } diff --git a/vendor/github.com/hashicorp/consul-template/version/version.go b/vendor/github.com/hashicorp/consul-template/version/version.go index 1796a33fa41..70bdbc86c51 100644 --- a/vendor/github.com/hashicorp/consul-template/version/version.go +++ b/vendor/github.com/hashicorp/consul-template/version/version.go @@ -2,7 +2,7 @@ package version import "fmt" -const Version = "0.19.5" +const Version = "0.20.0" var ( Name string diff --git a/vendor/github.com/hashicorp/consul-template/watch/watcher.go b/vendor/github.com/hashicorp/consul-template/watch/watcher.go index 6b863d8a8fb..fcbaa35217b 100644 --- a/vendor/github.com/hashicorp/consul-template/watch/watcher.go +++ b/vendor/github.com/hashicorp/consul-template/watch/watcher.go @@ -65,6 +65,9 @@ type NewWatcherInput struct { // VaultToken is the vault token to renew. VaultToken string + // VaultAgentTokenFile is the path to Vault Agent token file + VaultAgentTokenFile string + // RetryFuncs specify the different ways to retry based on the upstream. RetryFuncConsul RetryFunc RetryFuncDefault RetryFunc @@ -102,6 +105,16 @@ func NewWatcher(i *NewWatcherInput) (*Watcher, error) { } } + if len(i.VaultAgentTokenFile) > 0 { + vag, err := dep.NewVaultAgentTokenQuery(i.VaultAgentTokenFile) + if err != nil { + return nil, errors.Wrap(err, "watcher") + } + if _, err := w.Add(vag); err != nil { + return nil, errors.Wrap(err, "watcher") + } + } + return w, nil } diff --git a/vendor/github.com/hashicorp/consul/NOTICE.md b/vendor/github.com/hashicorp/consul/NOTICE.md new file mode 100644 index 00000000000..fe34b5e5715 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/NOTICE.md @@ -0,0 +1,3 @@ +Copyright © 2014-2018 HashiCorp, Inc. + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this project, you can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/vendor/github.com/hashicorp/consul/api/agent.go b/vendor/github.com/hashicorp/consul/api/agent.go index b42baed41d3..8cb81fc84e4 100644 --- a/vendor/github.com/hashicorp/consul/api/agent.go +++ b/vendor/github.com/hashicorp/consul/api/agent.go @@ -5,6 +5,39 @@ import ( "fmt" ) +// ServiceKind is the kind of service being registered. +type ServiceKind string + +const ( + // ServiceKindTypical is a typical, classic Consul service. This is + // represented by the absence of a value. This was chosen for ease of + // backwards compatibility: existing services in the catalog would + // default to the typical service. + ServiceKindTypical ServiceKind = "" + + // ServiceKindConnectProxy is a proxy for the Connect feature. This + // service proxies another service within Consul and speaks the connect + // protocol. + ServiceKindConnectProxy ServiceKind = "connect-proxy" +) + +// ProxyExecMode is the execution mode for a managed Connect proxy. +type ProxyExecMode string + +const ( + // ProxyExecModeDaemon indicates that the proxy command should be long-running + // and should be started and supervised by the agent until it's target service + // is deregistered. + ProxyExecModeDaemon ProxyExecMode = "daemon" + + // ProxyExecModeScript indicates that the proxy command should be invoke to + // completion on each change to the configuration of lifecycle event. The + // script typically fetches the config and certificates from the agent API and + // then configures an externally managed daemon, perhaps starting and stopping + // it if necessary. + ProxyExecModeScript ProxyExecMode = "script" +) + // AgentCheck represents a check known to the agent type AgentCheck struct { Node string @@ -20,14 +53,32 @@ type AgentCheck struct { // AgentService represents a service known to the agent type AgentService struct { + Kind ServiceKind ID string Service string Tags []string + Meta map[string]string Port int Address string EnableTagOverride bool CreateIndex uint64 ModifyIndex uint64 + ProxyDestination string + Connect *AgentServiceConnect +} + +// AgentServiceConnect represents the Connect configuration of a service. +type AgentServiceConnect struct { + Native bool + Proxy *AgentServiceConnectProxy +} + +// AgentServiceConnectProxy represents the Connect Proxy configuration of a +// service. +type AgentServiceConnectProxy struct { + ExecMode ProxyExecMode + Command []string + Config map[string]interface{} } // AgentMember represents a cluster member known to the agent @@ -60,6 +111,7 @@ type MembersOpts struct { // AgentServiceRegistration is used to register a new service type AgentServiceRegistration struct { + Kind ServiceKind `json:",omitempty"` ID string `json:",omitempty"` Name string `json:",omitempty"` Tags []string `json:",omitempty"` @@ -69,6 +121,8 @@ type AgentServiceRegistration struct { Meta map[string]string `json:",omitempty"` Check *AgentServiceCheck Checks AgentServiceChecks + ProxyDestination string `json:",omitempty"` + Connect *AgentServiceConnect `json:",omitempty"` } // AgentCheckRegistration is used to register a new check @@ -85,7 +139,6 @@ type AgentServiceCheck struct { CheckID string `json:",omitempty"` Name string `json:",omitempty"` Args []string `json:"ScriptArgs,omitempty"` - Script string `json:",omitempty"` // Deprecated, use Args. DockerContainerID string `json:",omitempty"` Shell string `json:",omitempty"` // Only supported for Docker. Interval string `json:",omitempty"` @@ -152,6 +205,31 @@ type SampledValue struct { Labels map[string]string } +// AgentAuthorizeParams are the request parameters for authorizing a request. +type AgentAuthorizeParams struct { + Target string + ClientCertURI string + ClientCertSerial string +} + +// AgentAuthorize is the response structure for Connect authorization. +type AgentAuthorize struct { + Authorized bool + Reason string +} + +// ConnectProxyConfig is the response structure for agent-local proxy +// configuration. +type ConnectProxyConfig struct { + ProxyServiceID string + TargetServiceID string + TargetServiceName string + ContentHash string + ExecMode ProxyExecMode + Command []string + Config map[string]interface{} +} + // Agent can be used to query the Agent endpoints type Agent struct { c *Client @@ -253,6 +331,7 @@ func (a *Agent) Services() (map[string]*AgentService, error) { if err := decodeBody(resp, &out); err != nil { return nil, err } + return out, nil } @@ -485,6 +564,91 @@ func (a *Agent) ForceLeave(node string) error { return nil } +// ConnectAuthorize is used to authorize an incoming connection +// to a natively integrated Connect service. +func (a *Agent) ConnectAuthorize(auth *AgentAuthorizeParams) (*AgentAuthorize, error) { + r := a.c.newRequest("POST", "/v1/agent/connect/authorize") + r.obj = auth + _, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var out AgentAuthorize + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ConnectCARoots returns the list of roots. +func (a *Agent) ConnectCARoots(q *QueryOptions) (*CARootList, *QueryMeta, error) { + r := a.c.newRequest("GET", "/v1/agent/connect/ca/roots") + r.setQueryOptions(q) + rtt, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out CARootList + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + +// ConnectCALeaf gets the leaf certificate for the given service ID. +func (a *Agent) ConnectCALeaf(serviceID string, q *QueryOptions) (*LeafCert, *QueryMeta, error) { + r := a.c.newRequest("GET", "/v1/agent/connect/ca/leaf/"+serviceID) + r.setQueryOptions(q) + rtt, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out LeafCert + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + +// ConnectProxyConfig gets the configuration for a local managed proxy instance. +// +// Note that this uses an unconventional blocking mechanism since it's +// agent-local state. That means there is no persistent raft index so we block +// based on object hash instead. +func (a *Agent) ConnectProxyConfig(proxyServiceID string, q *QueryOptions) (*ConnectProxyConfig, *QueryMeta, error) { + r := a.c.newRequest("GET", "/v1/agent/connect/proxy/"+proxyServiceID) + r.setQueryOptions(q) + rtt, resp, err := requireOK(a.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out ConnectProxyConfig + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + // EnableServiceMaintenance toggles service maintenance mode on // for the given service ID. func (a *Agent) EnableServiceMaintenance(serviceID, reason string) error { diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go index 1cdc21e3317..64923830296 100644 --- a/vendor/github.com/hashicorp/consul/api/api.go +++ b/vendor/github.com/hashicorp/consul/api/api.go @@ -82,6 +82,12 @@ type QueryOptions struct { // until the timeout or the next index is reached WaitIndex uint64 + // WaitHash is used by some endpoints instead of WaitIndex to perform blocking + // on state based on a hash of the response rather than a monotonic index. + // This is required when the state being blocked on is not stored in Raft, for + // example agent-local proxy configuration. + WaitHash string + // WaitTime is used to bound the duration of a wait. // Defaults to that of the Config, but can be overridden. WaitTime time.Duration @@ -106,6 +112,10 @@ type QueryOptions struct { // a value from 0 to 5 (inclusive). RelayFactor uint8 + // Connect filters prepared query execution to only include Connect-capable + // services. This currently affects prepared query execution. + Connect bool + // ctx is an optional context pass through to the underlying HTTP // request layer. Use Context() and WithContext() to manage this. ctx context.Context @@ -169,6 +179,11 @@ type QueryMeta struct { // a blocking query LastIndex uint64 + // LastContentHash. This can be used as a WaitHash to perform a blocking query + // for endpoints that support hash-based blocking. Endpoints that do not + // support it will return an empty hash. + LastContentHash string + // Time of last contact from the leader for the // server servicing the request LastContact time.Duration @@ -390,6 +405,29 @@ func SetupTLSConfig(tlsConfig *TLSConfig) (*tls.Config, error) { return tlsClientConfig, nil } +func (c *Config) GenerateEnv() []string { + env := make([]string, 0, 10) + + env = append(env, + fmt.Sprintf("%s=%s", HTTPAddrEnvName, c.Address), + fmt.Sprintf("%s=%s", HTTPTokenEnvName, c.Token), + fmt.Sprintf("%s=%t", HTTPSSLEnvName, c.Scheme == "https"), + fmt.Sprintf("%s=%s", HTTPCAFile, c.TLSConfig.CAFile), + fmt.Sprintf("%s=%s", HTTPCAPath, c.TLSConfig.CAPath), + fmt.Sprintf("%s=%s", HTTPClientCert, c.TLSConfig.CertFile), + fmt.Sprintf("%s=%s", HTTPClientKey, c.TLSConfig.KeyFile), + fmt.Sprintf("%s=%s", HTTPTLSServerName, c.TLSConfig.Address), + fmt.Sprintf("%s=%t", HTTPSSLVerifyEnvName, !c.TLSConfig.InsecureSkipVerify)) + + if c.HttpAuth != nil { + env = append(env, fmt.Sprintf("%s=%s:%s", HTTPAuthEnvName, c.HttpAuth.Username, c.HttpAuth.Password)) + } else { + env = append(env, fmt.Sprintf("%s=", HTTPAuthEnvName)) + } + + return env +} + // Client provides a client to the Consul API type Client struct { config Config @@ -533,6 +571,9 @@ func (r *request) setQueryOptions(q *QueryOptions) { if q.WaitTime != 0 { r.params.Set("wait", durToMsec(q.WaitTime)) } + if q.WaitHash != "" { + r.params.Set("hash", q.WaitHash) + } if q.Token != "" { r.header.Set("X-Consul-Token", q.Token) } @@ -547,6 +588,9 @@ func (r *request) setQueryOptions(q *QueryOptions) { if q.RelayFactor != 0 { r.params.Set("relay-factor", strconv.Itoa(int(q.RelayFactor))) } + if q.Connect { + r.params.Set("connect", "true") + } r.ctx = q.ctx } @@ -724,12 +768,16 @@ func (c *Client) write(endpoint string, in, out interface{}, q *WriteOptions) (* func parseQueryMeta(resp *http.Response, q *QueryMeta) error { header := resp.Header - // Parse the X-Consul-Index - index, err := strconv.ParseUint(header.Get("X-Consul-Index"), 10, 64) - if err != nil { - return fmt.Errorf("Failed to parse X-Consul-Index: %v", err) + // Parse the X-Consul-Index (if it's set - hash based blocking queries don't + // set this) + if indexStr := header.Get("X-Consul-Index"); indexStr != "" { + index, err := strconv.ParseUint(indexStr, 10, 64) + if err != nil { + return fmt.Errorf("Failed to parse X-Consul-Index: %v", err) + } + q.LastIndex = index } - q.LastIndex = index + q.LastContentHash = header.Get("X-Consul-ContentHash") // Parse the X-Consul-LastContact last, err := strconv.ParseUint(header.Get("X-Consul-LastContact"), 10, 64) diff --git a/vendor/github.com/hashicorp/consul/api/catalog.go b/vendor/github.com/hashicorp/consul/api/catalog.go index 80ce1bc8156..1a6bbc3b3f6 100644 --- a/vendor/github.com/hashicorp/consul/api/catalog.go +++ b/vendor/github.com/hashicorp/consul/api/catalog.go @@ -156,7 +156,20 @@ func (c *Catalog) Services(q *QueryOptions) (map[string][]string, *QueryMeta, er // Service is used to query catalog entries for a given service func (c *Catalog) Service(service, tag string, q *QueryOptions) ([]*CatalogService, *QueryMeta, error) { - r := c.c.newRequest("GET", "/v1/catalog/service/"+service) + return c.service(service, tag, q, false) +} + +// Connect is used to query catalog entries for a given Connect-enabled service +func (c *Catalog) Connect(service, tag string, q *QueryOptions) ([]*CatalogService, *QueryMeta, error) { + return c.service(service, tag, q, true) +} + +func (c *Catalog) service(service, tag string, q *QueryOptions, connect bool) ([]*CatalogService, *QueryMeta, error) { + path := "/v1/catalog/service/" + service + if connect { + path = "/v1/catalog/connect/" + service + } + r := c.c.newRequest("GET", path) r.setQueryOptions(q) if tag != "" { r.params.Set("tag", tag) diff --git a/vendor/github.com/hashicorp/consul/api/connect.go b/vendor/github.com/hashicorp/consul/api/connect.go new file mode 100644 index 00000000000..a40d1e2321a --- /dev/null +++ b/vendor/github.com/hashicorp/consul/api/connect.go @@ -0,0 +1,12 @@ +package api + +// Connect can be used to work with endpoints related to Connect, the +// feature for securely connecting services within Consul. +type Connect struct { + c *Client +} + +// Connect returns a handle to the connect-related endpoints +func (c *Client) Connect() *Connect { + return &Connect{c} +} diff --git a/vendor/github.com/hashicorp/consul/api/connect_ca.go b/vendor/github.com/hashicorp/consul/api/connect_ca.go new file mode 100644 index 00000000000..947f7097633 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/api/connect_ca.go @@ -0,0 +1,165 @@ +package api + +import ( + "fmt" + "time" + + "github.com/mitchellh/mapstructure" +) + +// CAConfig is the structure for the Connect CA configuration. +type CAConfig struct { + // Provider is the CA provider implementation to use. + Provider string + + // Configuration is arbitrary configuration for the provider. This + // should only contain primitive values and containers (such as lists + // and maps). + Config map[string]interface{} + + CreateIndex uint64 + ModifyIndex uint64 +} + +// ConsulCAProviderConfig is the config for the built-in Consul CA provider. +type ConsulCAProviderConfig struct { + PrivateKey string + RootCert string + RotationPeriod time.Duration +} + +// ParseConsulCAConfig takes a raw config map and returns a parsed +// ConsulCAProviderConfig. +func ParseConsulCAConfig(raw map[string]interface{}) (*ConsulCAProviderConfig, error) { + var config ConsulCAProviderConfig + decodeConf := &mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + ErrorUnused: true, + Result: &config, + WeaklyTypedInput: true, + } + + decoder, err := mapstructure.NewDecoder(decodeConf) + if err != nil { + return nil, err + } + + if err := decoder.Decode(raw); err != nil { + return nil, fmt.Errorf("error decoding config: %s", err) + } + + return &config, nil +} + +// CARootList is the structure for the results of listing roots. +type CARootList struct { + ActiveRootID string + TrustDomain string + Roots []*CARoot +} + +// CARoot represents a root CA certificate that is trusted. +type CARoot struct { + // ID is a globally unique ID (UUID) representing this CA root. + ID string + + // Name is a human-friendly name for this CA root. This value is + // opaque to Consul and is not used for anything internally. + Name string + + // RootCertPEM is the PEM-encoded public certificate. + RootCertPEM string `json:"RootCert"` + + // Active is true if this is the current active CA. This must only + // be true for exactly one CA. For any method that modifies roots in the + // state store, tests should be written to verify that multiple roots + // cannot be active. + Active bool + + CreateIndex uint64 + ModifyIndex uint64 +} + +// LeafCert is a certificate that has been issued by a Connect CA. +type LeafCert struct { + // SerialNumber is the unique serial number for this certificate. + // This is encoded in standard hex separated by :. + SerialNumber string + + // CertPEM and PrivateKeyPEM are the PEM-encoded certificate and private + // key for that cert, respectively. This should not be stored in the + // state store, but is present in the sign API response. + CertPEM string `json:",omitempty"` + PrivateKeyPEM string `json:",omitempty"` + + // Service is the name of the service for which the cert was issued. + // ServiceURI is the cert URI value. + Service string + ServiceURI string + + // ValidAfter and ValidBefore are the validity periods for the + // certificate. + ValidAfter time.Time + ValidBefore time.Time + + CreateIndex uint64 + ModifyIndex uint64 +} + +// CARoots queries the list of available roots. +func (h *Connect) CARoots(q *QueryOptions) (*CARootList, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/ca/roots") + r.setQueryOptions(q) + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out CARootList + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + +// CAGetConfig returns the current CA configuration. +func (h *Connect) CAGetConfig(q *QueryOptions) (*CAConfig, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/ca/configuration") + r.setQueryOptions(q) + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out CAConfig + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + +// CASetConfig sets the current CA configuration. +func (h *Connect) CASetConfig(conf *CAConfig, q *WriteOptions) (*WriteMeta, error) { + r := h.c.newRequest("PUT", "/v1/connect/ca/configuration") + r.setWriteOptions(q) + r.obj = conf + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + wm := &WriteMeta{} + wm.RequestTime = rtt + return wm, nil +} diff --git a/vendor/github.com/hashicorp/consul/api/connect_intention.go b/vendor/github.com/hashicorp/consul/api/connect_intention.go new file mode 100644 index 00000000000..a996c03e5e7 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/api/connect_intention.go @@ -0,0 +1,302 @@ +package api + +import ( + "bytes" + "fmt" + "io" + "time" +) + +// Intention defines an intention for the Connect Service Graph. This defines +// the allowed or denied behavior of a connection between two services using +// Connect. +type Intention struct { + // ID is the UUID-based ID for the intention, always generated by Consul. + ID string + + // Description is a human-friendly description of this intention. + // It is opaque to Consul and is only stored and transferred in API + // requests. + Description string + + // SourceNS, SourceName are the namespace and name, respectively, of + // the source service. Either of these may be the wildcard "*", but only + // the full value can be a wildcard. Partial wildcards are not allowed. + // The source may also be a non-Consul service, as specified by SourceType. + // + // DestinationNS, DestinationName is the same, but for the destination + // service. The same rules apply. The destination is always a Consul + // service. + SourceNS, SourceName string + DestinationNS, DestinationName string + + // SourceType is the type of the value for the source. + SourceType IntentionSourceType + + // Action is whether this is a whitelist or blacklist intention. + Action IntentionAction + + // DefaultAddr, DefaultPort of the local listening proxy (if any) to + // make this connection. + DefaultAddr string + DefaultPort int + + // Meta is arbitrary metadata associated with the intention. This is + // opaque to Consul but is served in API responses. + Meta map[string]string + + // Precedence is the order that the intention will be applied, with + // larger numbers being applied first. This is a read-only field, on + // any intention update it is updated. + Precedence int + + // CreatedAt and UpdatedAt keep track of when this record was created + // or modified. + CreatedAt, UpdatedAt time.Time + + CreateIndex uint64 + ModifyIndex uint64 +} + +// String returns human-friendly output describing ths intention. +func (i *Intention) String() string { + return fmt.Sprintf("%s => %s (%s)", + i.SourceString(), + i.DestinationString(), + i.Action) +} + +// SourceString returns the namespace/name format for the source, or +// just "name" if the namespace is the default namespace. +func (i *Intention) SourceString() string { + return i.partString(i.SourceNS, i.SourceName) +} + +// DestinationString returns the namespace/name format for the source, or +// just "name" if the namespace is the default namespace. +func (i *Intention) DestinationString() string { + return i.partString(i.DestinationNS, i.DestinationName) +} + +func (i *Intention) partString(ns, n string) string { + // For now we omit the default namespace from the output. In the future + // we might want to look at this and show this in a multi-namespace world. + if ns != "" && ns != IntentionDefaultNamespace { + n = ns + "/" + n + } + + return n +} + +// IntentionDefaultNamespace is the default namespace value. +const IntentionDefaultNamespace = "default" + +// IntentionAction is the action that the intention represents. This +// can be "allow" or "deny" to whitelist or blacklist intentions. +type IntentionAction string + +const ( + IntentionActionAllow IntentionAction = "allow" + IntentionActionDeny IntentionAction = "deny" +) + +// IntentionSourceType is the type of the source within an intention. +type IntentionSourceType string + +const ( + // IntentionSourceConsul is a service within the Consul catalog. + IntentionSourceConsul IntentionSourceType = "consul" +) + +// IntentionMatch are the arguments for the intention match API. +type IntentionMatch struct { + By IntentionMatchType + Names []string +} + +// IntentionMatchType is the target for a match request. For example, +// matching by source will look for all intentions that match the given +// source value. +type IntentionMatchType string + +const ( + IntentionMatchSource IntentionMatchType = "source" + IntentionMatchDestination IntentionMatchType = "destination" +) + +// IntentionCheck are the arguments for the intention check API. For +// more documentation see the IntentionCheck function. +type IntentionCheck struct { + // Source and Destination are the source and destination values to + // check. The destination is always a Consul service, but the source + // may be other values as defined by the SourceType. + Source, Destination string + + // SourceType is the type of the value for the source. + SourceType IntentionSourceType +} + +// Intentions returns the list of intentions. +func (h *Connect) Intentions(q *QueryOptions) ([]*Intention, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/intentions") + r.setQueryOptions(q) + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out []*Intention + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return out, qm, nil +} + +// IntentionGet retrieves a single intention. +func (h *Connect) IntentionGet(id string, q *QueryOptions) (*Intention, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/intentions/"+id) + r.setQueryOptions(q) + rtt, resp, err := h.c.doRequest(r) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + if resp.StatusCode == 404 { + return nil, qm, nil + } else if resp.StatusCode != 200 { + var buf bytes.Buffer + io.Copy(&buf, resp.Body) + return nil, nil, fmt.Errorf( + "Unexpected response %d: %s", resp.StatusCode, buf.String()) + } + + var out Intention + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return &out, qm, nil +} + +// IntentionDelete deletes a single intention. +func (h *Connect) IntentionDelete(id string, q *WriteOptions) (*WriteMeta, error) { + r := h.c.newRequest("DELETE", "/v1/connect/intentions/"+id) + r.setWriteOptions(q) + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + qm := &WriteMeta{} + qm.RequestTime = rtt + + return qm, nil +} + +// IntentionMatch returns the list of intentions that match a given source +// or destination. The returned intentions are ordered by precedence where +// result[0] is the highest precedence (if that matches, then that rule overrides +// all other rules). +// +// Matching can be done for multiple names at the same time. The resulting +// map is keyed by the given names. Casing is preserved. +func (h *Connect) IntentionMatch(args *IntentionMatch, q *QueryOptions) (map[string][]*Intention, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/intentions/match") + r.setQueryOptions(q) + r.params.Set("by", string(args.By)) + for _, name := range args.Names { + r.params.Add("name", name) + } + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out map[string][]*Intention + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return out, qm, nil +} + +// IntentionCheck returns whether a given source/destination would be allowed +// or not given the current set of intentions and the configuration of Consul. +func (h *Connect) IntentionCheck(args *IntentionCheck, q *QueryOptions) (bool, *QueryMeta, error) { + r := h.c.newRequest("GET", "/v1/connect/intentions/check") + r.setQueryOptions(q) + r.params.Set("source", args.Source) + r.params.Set("destination", args.Destination) + if args.SourceType != "" { + r.params.Set("source-type", string(args.SourceType)) + } + rtt, resp, err := requireOK(h.c.doRequest(r)) + if err != nil { + return false, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out struct{ Allowed bool } + if err := decodeBody(resp, &out); err != nil { + return false, nil, err + } + return out.Allowed, qm, nil +} + +// IntentionCreate will create a new intention. The ID in the given +// structure must be empty and a generate ID will be returned on +// success. +func (c *Connect) IntentionCreate(ixn *Intention, q *WriteOptions) (string, *WriteMeta, error) { + r := c.c.newRequest("POST", "/v1/connect/intentions") + r.setWriteOptions(q) + r.obj = ixn + rtt, resp, err := requireOK(c.c.doRequest(r)) + if err != nil { + return "", nil, err + } + defer resp.Body.Close() + + wm := &WriteMeta{} + wm.RequestTime = rtt + + var out struct{ ID string } + if err := decodeBody(resp, &out); err != nil { + return "", nil, err + } + return out.ID, wm, nil +} + +// IntentionUpdate will update an existing intention. The ID in the given +// structure must be non-empty. +func (c *Connect) IntentionUpdate(ixn *Intention, q *WriteOptions) (*WriteMeta, error) { + r := c.c.newRequest("PUT", "/v1/connect/intentions/"+ixn.ID) + r.setWriteOptions(q) + r.obj = ixn + rtt, resp, err := requireOK(c.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + wm := &WriteMeta{} + wm.RequestTime = rtt + return wm, nil +} diff --git a/vendor/github.com/hashicorp/consul/api/health.go b/vendor/github.com/hashicorp/consul/api/health.go index 53f3de4f795..1835da559f0 100644 --- a/vendor/github.com/hashicorp/consul/api/health.go +++ b/vendor/github.com/hashicorp/consul/api/health.go @@ -159,7 +159,24 @@ func (h *Health) Checks(service string, q *QueryOptions) (HealthChecks, *QueryMe // for a given service. It can optionally do server-side filtering on a tag // or nodes with passing health checks only. func (h *Health) Service(service, tag string, passingOnly bool, q *QueryOptions) ([]*ServiceEntry, *QueryMeta, error) { - r := h.c.newRequest("GET", "/v1/health/service/"+service) + return h.service(service, tag, passingOnly, q, false) +} + +// Connect is equivalent to Service except that it will only return services +// which are Connect-enabled and will returns the connection address for Connect +// client's to use which may be a proxy in front of the named service. If +// passingOnly is true only instances where both the service and any proxy are +// healthy will be returned. +func (h *Health) Connect(service, tag string, passingOnly bool, q *QueryOptions) ([]*ServiceEntry, *QueryMeta, error) { + return h.service(service, tag, passingOnly, q, true) +} + +func (h *Health) service(service, tag string, passingOnly bool, q *QueryOptions, connect bool) ([]*ServiceEntry, *QueryMeta, error) { + path := "/v1/health/service/" + service + if connect { + path = "/v1/health/connect/" + service + } + r := h.c.newRequest("GET", path) r.setQueryOptions(q) if tag != "" { r.params.Set("tag", tag) diff --git a/vendor/github.com/hashicorp/consul/api/lock.go b/vendor/github.com/hashicorp/consul/api/lock.go index 41f72e7d23a..82339cb744a 100644 --- a/vendor/github.com/hashicorp/consul/api/lock.go +++ b/vendor/github.com/hashicorp/consul/api/lock.go @@ -181,11 +181,12 @@ WAIT: // Handle the one-shot mode. if l.opts.LockTryOnce && attempts > 0 { elapsed := time.Since(start) - if elapsed > qOpts.WaitTime { + if elapsed > l.opts.LockWaitTime { return nil, nil } - qOpts.WaitTime -= elapsed + // Query wait time should not exceed the lock wait time + qOpts.WaitTime = l.opts.LockWaitTime - elapsed } attempts++ diff --git a/vendor/github.com/hashicorp/consul/api/prepared_query.go b/vendor/github.com/hashicorp/consul/api/prepared_query.go index d322dd86794..8bb1004eec7 100644 --- a/vendor/github.com/hashicorp/consul/api/prepared_query.go +++ b/vendor/github.com/hashicorp/consul/api/prepared_query.go @@ -54,6 +54,14 @@ type ServiceQuery struct { // pair is in this map it must be present on the node in order for the // service entry to be returned. NodeMeta map[string]string + + // Connect if true will filter the prepared query results to only + // include Connect-capable services. These include both native services + // and proxies for matching services. Note that if a proxy matches, + // the constraints in the query above (Near, OnlyPassing, etc.) apply + // to the _proxy_ and not the service being proxied. In practice, proxies + // should be directly next to their services so this isn't an issue. + Connect bool } // QueryTemplate carries the arguments for creating a templated query. diff --git a/vendor/github.com/hashicorp/consul/api/semaphore.go b/vendor/github.com/hashicorp/consul/api/semaphore.go index d0c57417788..bc4f885fec0 100644 --- a/vendor/github.com/hashicorp/consul/api/semaphore.go +++ b/vendor/github.com/hashicorp/consul/api/semaphore.go @@ -199,11 +199,12 @@ WAIT: // Handle the one-shot mode. if s.opts.SemaphoreTryOnce && attempts > 0 { elapsed := time.Since(start) - if elapsed > qOpts.WaitTime { + if elapsed > s.opts.SemaphoreWaitTime { return nil, nil } - qOpts.WaitTime -= elapsed + // Query wait time should not exceed the semaphore wait time + qOpts.WaitTime = s.opts.SemaphoreWaitTime - elapsed } attempts++ diff --git a/vendor/github.com/hashicorp/go-retryablehttp/client.go b/vendor/github.com/hashicorp/go-retryablehttp/client.go index 21f45e5ed64..15f1e8850e0 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/client.go +++ b/vendor/github.com/hashicorp/go-retryablehttp/client.go @@ -36,7 +36,7 @@ import ( "strings" "time" - "github.com/hashicorp/go-cleanhttp" + cleanhttp "github.com/hashicorp/go-cleanhttp" ) var ( @@ -81,6 +81,28 @@ func (r *Request) WithContext(ctx context.Context) *Request { return r } +// BodyBytes allows accessing the request body. It is an analogue to +// http.Request's Body variable, but it returns a copy of the underlying data +// rather than consuming it. +// +// This function is not thread-safe; do not call it at the same time as another +// call, or at the same time this request is being used with Client.Do. +func (r *Request) BodyBytes() ([]byte, error) { + if r.body == nil { + return nil, nil + } + body, err := r.body() + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(body) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + // NewRequest creates a new wrapped request. func NewRequest(method, url string, rawBody interface{}) (*Request, error) { var err error @@ -183,18 +205,24 @@ func NewRequest(method, url string, rawBody interface{}) (*Request, error) { return &Request{body, httpReq}, nil } +// Logger interface allows to use other loggers than +// standard log.Logger. +type Logger interface { + Printf(string, ...interface{}) +} + // RequestLogHook allows a function to run before each retry. The HTTP // request which will be made, and the retry number (0 for the initial // request) are available to users. The internal logger is exposed to // consumers. -type RequestLogHook func(*log.Logger, *http.Request, int) +type RequestLogHook func(Logger, *http.Request, int) // ResponseLogHook is like RequestLogHook, but allows running a function // on each HTTP response. This function will be invoked at the end of // every HTTP request executed, regardless of whether a subsequent retry // needs to be performed or not. If the response body is read or closed // from this method, this will affect the response returned from Do(). -type ResponseLogHook func(*log.Logger, *http.Response) +type ResponseLogHook func(Logger, *http.Response) // CheckRetry specifies a policy for handling retries. It is called // following each request with the response and error values returned by @@ -221,7 +249,7 @@ type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Resp // like automatic retries to tolerate minor outages. type Client struct { HTTPClient *http.Client // Internal HTTP client. - Logger *log.Logger // Customer logger instance. + Logger Logger // Customer logger instance. RetryWaitMin time.Duration // Minimum time to wait RetryWaitMax time.Duration // Maximum time to wait diff --git a/vendor/github.com/hashicorp/go-retryablehttp/go.mod b/vendor/github.com/hashicorp/go-retryablehttp/go.mod new file mode 100644 index 00000000000..d28c8c8eb6c --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/go.mod @@ -0,0 +1,3 @@ +module github.com/hashicorp/go-retryablehttp + +require github.com/hashicorp/go-cleanhttp v0.5.0 diff --git a/vendor/github.com/hashicorp/go-retryablehttp/go.sum b/vendor/github.com/hashicorp/go-retryablehttp/go.sum new file mode 100644 index 00000000000..3ed0fd98e92 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/go.sum @@ -0,0 +1,2 @@ +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= diff --git a/vendor/github.com/hashicorp/vault/api/auth_token.go b/vendor/github.com/hashicorp/vault/api/auth_token.go index c66fba348a2..ed594eee852 100644 --- a/vendor/github.com/hashicorp/vault/api/auth_token.go +++ b/vendor/github.com/hashicorp/vault/api/auth_token.go @@ -271,4 +271,5 @@ type TokenCreateRequest struct { DisplayName string `json:"display_name"` NumUses int `json:"num_uses"` Renewable *bool `json:"renewable,omitempty"` + Type string `json:"type"` } diff --git a/vendor/github.com/hashicorp/vault/api/client.go b/vendor/github.com/hashicorp/vault/api/client.go index c7ced823738..80ccd7d5029 100644 --- a/vendor/github.com/hashicorp/vault/api/client.go +++ b/vendor/github.com/hashicorp/vault/api/client.go @@ -16,9 +16,9 @@ import ( "unicode" "github.com/hashicorp/errwrap" - "github.com/hashicorp/go-cleanhttp" + cleanhttp "github.com/hashicorp/go-cleanhttp" retryablehttp "github.com/hashicorp/go-retryablehttp" - "github.com/hashicorp/go-rootcerts" + rootcerts "github.com/hashicorp/go-rootcerts" "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/parseutil" "golang.org/x/net/http2" @@ -84,6 +84,14 @@ type Config struct { // then that limiter will be used. Note that an empty Limiter // is equivalent blocking all events. Limiter *rate.Limiter + + // OutputCurlString causes the actual request to return an error of type + // *OutputStringError. Type asserting the error message will allow + // fetching a cURL-compatible string for the operation. + // + // Note: It is not thread-safe to set this and make concurrent requests + // with the same client. Cloning a client will not clone this value. + OutputCurlString bool } // TLSConfig contains the parameters needed to configure TLS on the HTTP client @@ -121,7 +129,7 @@ type TLSConfig struct { func DefaultConfig() *Config { config := &Config{ Address: "https://127.0.0.1:8200", - HttpClient: cleanhttp.DefaultClient(), + HttpClient: cleanhttp.DefaultPooledClient(), } config.HttpClient.Timeout = time.Second * 60 @@ -438,6 +446,24 @@ func (c *Client) SetClientTimeout(timeout time.Duration) { c.config.Timeout = timeout } +func (c *Client) OutputCurlString() bool { + c.modifyLock.RLock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + c.modifyLock.RUnlock() + + return c.config.OutputCurlString +} + +func (c *Client) SetOutputCurlString(curl bool) { + c.modifyLock.RLock() + c.config.modifyLock.Lock() + defer c.config.modifyLock.Unlock() + c.modifyLock.RUnlock() + + c.config.OutputCurlString = curl +} + // CurrentWrappingLookupFunc sets a lookup function that returns desired wrap TTLs // for a given operation and path func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc { @@ -546,6 +572,10 @@ func (c *Client) SetBackoff(backoff retryablehttp.Backoff) { // underlying http.Client is used; modifying the client from more than one // goroutine at once may not be safe, so modify the client as needed and then // clone. +// +// Also, only the client's config is currently copied; this means items not in +// the api.Config struct, such as policy override and wrapping function +// behavior, must currently then be set as desired on the new client. func (c *Client) Clone() (*Client, error) { c.modifyLock.RLock() c.config.modifyLock.RLock() @@ -658,6 +688,7 @@ func (c *Client) RawRequestWithContext(ctx context.Context, r *Request) (*Respon backoff := c.config.Backoff httpClient := c.config.HttpClient timeout := c.config.Timeout + outputCurlString := c.config.OutputCurlString c.config.modifyLock.RUnlock() c.modifyLock.RUnlock() @@ -684,6 +715,11 @@ START: return nil, fmt.Errorf("nil request created") } + if outputCurlString { + LastOutputStringError = &OutputStringError{Request: req} + return nil, LastOutputStringError + } + if timeout != 0 { ctx, _ = context.WithTimeout(ctx, timeout) } diff --git a/vendor/github.com/hashicorp/vault/api/output_string.go b/vendor/github.com/hashicorp/vault/api/output_string.go new file mode 100644 index 00000000000..ebfdad6c0da --- /dev/null +++ b/vendor/github.com/hashicorp/vault/api/output_string.go @@ -0,0 +1,69 @@ +package api + +import ( + "fmt" + "strings" + + retryablehttp "github.com/hashicorp/go-retryablehttp" +) + +const ( + ErrOutputStringRequest = "output a string, please" +) + +var ( + LastOutputStringError *OutputStringError +) + +type OutputStringError struct { + *retryablehttp.Request + parsingError error + parsedCurlString string +} + +func (d *OutputStringError) Error() string { + if d.parsedCurlString == "" { + d.parseRequest() + if d.parsingError != nil { + return d.parsingError.Error() + } + } + + return ErrOutputStringRequest +} + +func (d *OutputStringError) parseRequest() { + body, err := d.Request.BodyBytes() + if err != nil { + d.parsingError = err + return + } + + // Build cURL string + d.parsedCurlString = "curl " + d.parsedCurlString = fmt.Sprintf("%s-X %s ", d.parsedCurlString, d.Request.Method) + for k, v := range d.Request.Header { + for _, h := range v { + if strings.ToLower(k) == "x-vault-token" { + h = `$(vault print token)` + } + d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\" ", d.parsedCurlString, k, h) + } + } + + if len(body) > 0 { + // We need to escape single quotes since that's what we're using to + // quote the body + escapedBody := strings.Replace(string(body), "'", "'\"'\"'", -1) + d.parsedCurlString = fmt.Sprintf("%s-d '%s' ", d.parsedCurlString, escapedBody) + } + + d.parsedCurlString = fmt.Sprintf("%s%s", d.parsedCurlString, d.Request.URL.String()) +} + +func (d *OutputStringError) CurlString() string { + if d.parsedCurlString == "" { + d.parseRequest() + } + return d.parsedCurlString +} diff --git a/vendor/github.com/hashicorp/vault/api/request.go b/vendor/github.com/hashicorp/vault/api/request.go index 5bcff8c6c51..4efa2aa8417 100644 --- a/vendor/github.com/hashicorp/vault/api/request.go +++ b/vendor/github.com/hashicorp/vault/api/request.go @@ -8,6 +8,8 @@ import ( "net/http" "net/url" + "github.com/hashicorp/vault/helper/consts" + retryablehttp "github.com/hashicorp/go-retryablehttp" ) @@ -124,7 +126,7 @@ func (r *Request) toRetryableHTTP() (*retryablehttp.Request, error) { } if len(r.ClientToken) != 0 { - req.Header.Set("X-Vault-Token", r.ClientToken) + req.Header.Set(consts.AuthHeaderName, r.ClientToken) } if len(r.WrapTTL) != 0 { diff --git a/vendor/github.com/hashicorp/vault/api/ssh_agent.go b/vendor/github.com/hashicorp/vault/api/ssh_agent.go index 1dd681a5d49..b8e82224cae 100644 --- a/vendor/github.com/hashicorp/vault/api/ssh_agent.go +++ b/vendor/github.com/hashicorp/vault/api/ssh_agent.go @@ -9,9 +9,9 @@ import ( "os" "github.com/hashicorp/errwrap" - "github.com/hashicorp/go-cleanhttp" - "github.com/hashicorp/go-multierror" - "github.com/hashicorp/go-rootcerts" + cleanhttp "github.com/hashicorp/go-cleanhttp" + multierror "github.com/hashicorp/go-multierror" + rootcerts "github.com/hashicorp/go-rootcerts" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/vault/helper/hclutil" diff --git a/vendor/github.com/hashicorp/vault/api/sys_auth.go b/vendor/github.com/hashicorp/vault/api/sys_auth.go index 447c5d54b76..e7a9c222d8f 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_auth.go +++ b/vendor/github.com/hashicorp/vault/api/sys_auth.go @@ -73,46 +73,8 @@ func (c *Sys) DisableAuth(path string) error { return err } -// Structures for the requests/resposne are all down here. They aren't -// individually documented because the map almost directly to the raw HTTP API -// documentation. Please refer to that documentation for more details. - -type EnableAuthOptions struct { - Type string `json:"type"` - Description string `json:"description"` - Config AuthConfigInput `json:"config"` - Local bool `json:"local"` - PluginName string `json:"plugin_name,omitempty"` - SealWrap bool `json:"seal_wrap" mapstructure:"seal_wrap"` - Options map[string]string `json:"options" mapstructure:"options"` -} - -type AuthConfigInput struct { - DefaultLeaseTTL string `json:"default_lease_ttl" mapstructure:"default_lease_ttl"` - MaxLeaseTTL string `json:"max_lease_ttl" mapstructure:"max_lease_ttl"` - PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` - AuditNonHMACRequestKeys []string `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"` - AuditNonHMACResponseKeys []string `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"` - ListingVisibility string `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"` - PassthroughRequestHeaders []string `json:"passthrough_request_headers,omitempty" mapstructure:"passthrough_request_headers"` -} - -type AuthMount struct { - Type string `json:"type" mapstructure:"type"` - Description string `json:"description" mapstructure:"description"` - Accessor string `json:"accessor" mapstructure:"accessor"` - Config AuthConfigOutput `json:"config" mapstructure:"config"` - Local bool `json:"local" mapstructure:"local"` - SealWrap bool `json:"seal_wrap" mapstructure:"seal_wrap"` - Options map[string]string `json:"options" mapstructure:"options"` -} - -type AuthConfigOutput struct { - DefaultLeaseTTL int `json:"default_lease_ttl" mapstructure:"default_lease_ttl"` - MaxLeaseTTL int `json:"max_lease_ttl" mapstructure:"max_lease_ttl"` - PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` - AuditNonHMACRequestKeys []string `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"` - AuditNonHMACResponseKeys []string `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"` - ListingVisibility string `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"` - PassthroughRequestHeaders []string `json:"passthrough_request_headers,omitempty" mapstructure:"passthrough_request_headers"` -} +// Rather than duplicate, we can use modern Go's type aliasing +type EnableAuthOptions = MountInput +type AuthConfigInput = MountConfigInput +type AuthMount = MountOutput +type AuthConfigOutput = MountConfigOutput diff --git a/vendor/github.com/hashicorp/vault/api/sys_health.go b/vendor/github.com/hashicorp/vault/api/sys_health.go index bd74e8269df..d5d7796008f 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_health.go +++ b/vendor/github.com/hashicorp/vault/api/sys_health.go @@ -30,10 +30,12 @@ type HealthResponse struct { Initialized bool `json:"initialized"` Sealed bool `json:"sealed"` Standby bool `json:"standby"` + PerformanceStandby bool `json:"performance_standby"` ReplicationPerformanceMode string `json:"replication_performance_mode"` ReplicationDRMode string `json:"replication_dr_mode"` ServerTimeUTC int64 `json:"server_time_utc"` Version string `json:"version"` ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` + LastWAL uint64 `json:"last_wal,omitempty"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_leader.go b/vendor/github.com/hashicorp/vault/api/sys_leader.go index dfef8345cc5..8846dcdfae9 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_leader.go +++ b/vendor/github.com/hashicorp/vault/api/sys_leader.go @@ -25,4 +25,5 @@ type LeaderResponse struct { LeaderClusterAddress string `json:"leader_cluster_address"` PerfStandby bool `json:"performance_standby"` PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"` + LastWAL uint64 `json:"last_wal"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_mounts.go b/vendor/github.com/hashicorp/vault/api/sys_mounts.go index 8a32b095e6a..993a1b67bfe 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_mounts.go +++ b/vendor/github.com/hashicorp/vault/api/sys_mounts.go @@ -132,10 +132,13 @@ type MountInput struct { Type string `json:"type"` Description string `json:"description"` Config MountConfigInput `json:"config"` - Options map[string]string `json:"options"` Local bool `json:"local"` - PluginName string `json:"plugin_name,omitempty"` SealWrap bool `json:"seal_wrap" mapstructure:"seal_wrap"` + Options map[string]string `json:"options"` + + // Deprecated: Newer server responses should be returning this information in the + // Type field (json: "type") instead. + PluginName string `json:"plugin_name,omitempty"` } type MountConfigInput struct { @@ -144,11 +147,15 @@ type MountConfigInput struct { Description *string `json:"description,omitempty" mapstructure:"description"` MaxLeaseTTL string `json:"max_lease_ttl" mapstructure:"max_lease_ttl"` ForceNoCache bool `json:"force_no_cache" mapstructure:"force_no_cache"` - PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` AuditNonHMACRequestKeys []string `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"` AuditNonHMACResponseKeys []string `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"` ListingVisibility string `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"` PassthroughRequestHeaders []string `json:"passthrough_request_headers,omitempty" mapstructure:"passthrough_request_headers"` + AllowedResponseHeaders []string `json:"allowed_response_headers,omitempty" mapstructure:"allowed_response_headers"` + TokenType string `json:"token_type,omitempty" mapstructure:"token_type"` + + // Deprecated: This field will always be blank for newer server responses. + PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` } type MountOutput struct { @@ -165,9 +172,13 @@ type MountConfigOutput struct { DefaultLeaseTTL int `json:"default_lease_ttl" mapstructure:"default_lease_ttl"` MaxLeaseTTL int `json:"max_lease_ttl" mapstructure:"max_lease_ttl"` ForceNoCache bool `json:"force_no_cache" mapstructure:"force_no_cache"` - PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` AuditNonHMACRequestKeys []string `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"` AuditNonHMACResponseKeys []string `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"` ListingVisibility string `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"` PassthroughRequestHeaders []string `json:"passthrough_request_headers,omitempty" mapstructure:"passthrough_request_headers"` + AllowedResponseHeaders []string `json:"allowed_response_headers,omitempty" mapstructure:"allowed_response_headers"` + TokenType string `json:"token_type,omitempty" mapstructure:"token_type"` + + // Deprecated: This field will always be blank for newer server responses. + PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_plugins.go b/vendor/github.com/hashicorp/vault/api/sys_plugins.go index b2f18d94d76..ebfcfbfc022 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_plugins.go +++ b/vendor/github.com/hashicorp/vault/api/sys_plugins.go @@ -2,48 +2,135 @@ package api import ( "context" + "errors" "fmt" "net/http" + + "github.com/hashicorp/vault/helper/consts" + "github.com/mitchellh/mapstructure" ) // ListPluginsInput is used as input to the ListPlugins function. -type ListPluginsInput struct{} +type ListPluginsInput struct { + // Type of the plugin. Required. + Type consts.PluginType `json:"type"` +} // ListPluginsResponse is the response from the ListPlugins call. type ListPluginsResponse struct { + // PluginsByType is the list of plugins by type. + PluginsByType map[consts.PluginType][]string `json:"types"` + // Names is the list of names of the plugins. + // + // Deprecated: Newer server responses should be returning PluginsByType (json: + // "types") instead. Names []string `json:"names"` } // ListPlugins lists all plugins in the catalog and returns their names as a // list of strings. func (c *Sys) ListPlugins(i *ListPluginsInput) (*ListPluginsResponse, error) { - path := "/v1/sys/plugins/catalog" - req := c.c.NewRequest("LIST", path) + path := "" + method := "" + if i.Type == consts.PluginTypeUnknown { + path = "/v1/sys/plugins/catalog" + method = "GET" + } else { + path = fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Type) + method = "LIST" + } + + req := c.c.NewRequest(method, path) + if method == "LIST" { + // Set this for broader compatibility, but we use LIST above to be able + // to handle the wrapping lookup function + req.Method = "GET" + req.Params.Set("list", "true") + } ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() resp, err := c.c.RawRequestWithContext(ctx, req) - if err != nil { + if err != nil && resp == nil { return nil, err } + if resp == nil { + return nil, nil + } defer resp.Body.Close() - var result struct { - Data struct { - Keys []string `json:"keys"` - } `json:"data"` + // We received an Unsupported Operation response from Vault, indicating + // Vault of an older version that doesn't support the GET method yet; + // switch it to a LIST. + if resp.StatusCode == 405 { + req.Params.Set("list", "true") + resp, err := c.c.RawRequestWithContext(ctx, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result struct { + Data struct { + Keys []string `json:"keys"` + } `json:"data"` + } + if err := resp.DecodeJSON(&result); err != nil { + return nil, err + } + return &ListPluginsResponse{Names: result.Data.Keys}, nil } - if err := resp.DecodeJSON(&result); err != nil { + + secret, err := ParseSecret(resp.Body) + if err != nil { return nil, err } + if secret == nil || secret.Data == nil { + return nil, errors.New("data from server response is empty") + } - return &ListPluginsResponse{Names: result.Data.Keys}, nil + result := &ListPluginsResponse{ + PluginsByType: make(map[consts.PluginType][]string), + } + if i.Type == consts.PluginTypeUnknown { + for pluginTypeStr, pluginsRaw := range secret.Data { + pluginType, err := consts.ParsePluginType(pluginTypeStr) + if err != nil { + return nil, err + } + + pluginsIfc, ok := pluginsRaw.([]interface{}) + if !ok { + return nil, fmt.Errorf("unable to parse plugins for %q type", pluginTypeStr) + } + + plugins := make([]string, len(pluginsIfc)) + for i, nameIfc := range pluginsIfc { + name, ok := nameIfc.(string) + if !ok { + + } + plugins[i] = name + } + result.PluginsByType[pluginType] = plugins + } + } else { + var respKeys []string + if err := mapstructure.Decode(secret.Data["keys"], &respKeys); err != nil { + return nil, err + } + result.PluginsByType[i.Type] = respKeys + } + + return result, nil } // GetPluginInput is used as input to the GetPlugin function. type GetPluginInput struct { Name string `json:"-"` + + // Type of the plugin. Required. + Type consts.PluginType `json:"type"` } // GetPluginResponse is the response from the GetPlugin call. @@ -55,8 +142,9 @@ type GetPluginResponse struct { SHA256 string `json:"sha256"` } +// GetPlugin retrieves information about the plugin. func (c *Sys) GetPlugin(i *GetPluginInput) (*GetPluginResponse, error) { - path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name) + path := catalogPathByType(i.Type, i.Name) req := c.c.NewRequest(http.MethodGet, path) ctx, cancelFunc := context.WithCancel(context.Background()) @@ -68,13 +156,13 @@ func (c *Sys) GetPlugin(i *GetPluginInput) (*GetPluginResponse, error) { defer resp.Body.Close() var result struct { - Data GetPluginResponse + Data *GetPluginResponse } err = resp.DecodeJSON(&result) if err != nil { return nil, err } - return &result.Data, err + return result.Data, err } // RegisterPluginInput is used as input to the RegisterPlugin function. @@ -82,6 +170,9 @@ type RegisterPluginInput struct { // Name is the name of the plugin. Required. Name string `json:"-"` + // Type of the plugin. Required. + Type consts.PluginType `json:"type"` + // Args is the list of args to spawn the process with. Args []string `json:"args,omitempty"` @@ -94,8 +185,9 @@ type RegisterPluginInput struct { // RegisterPlugin registers the plugin with the given information. func (c *Sys) RegisterPlugin(i *RegisterPluginInput) error { - path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name) + path := catalogPathByType(i.Type, i.Name) req := c.c.NewRequest(http.MethodPut, path) + if err := req.SetJSONBody(i); err != nil { return err } @@ -113,12 +205,15 @@ func (c *Sys) RegisterPlugin(i *RegisterPluginInput) error { type DeregisterPluginInput struct { // Name is the name of the plugin. Required. Name string `json:"-"` + + // Type of the plugin. Required. + Type consts.PluginType `json:"type"` } // DeregisterPlugin removes the plugin with the given name from the plugin // catalog. func (c *Sys) DeregisterPlugin(i *DeregisterPluginInput) error { - path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name) + path := catalogPathByType(i.Type, i.Name) req := c.c.NewRequest(http.MethodDelete, path) ctx, cancelFunc := context.WithCancel(context.Background()) @@ -129,3 +224,15 @@ func (c *Sys) DeregisterPlugin(i *DeregisterPluginInput) error { } return err } + +// catalogPathByType is a helper to construct the proper API path by plugin type +func catalogPathByType(pluginType consts.PluginType, name string) string { + path := fmt.Sprintf("/v1/sys/plugins/catalog/%s/%s", pluginType, name) + + // Backwards compat, if type is not provided then use old path + if pluginType == consts.PluginTypeUnknown { + path = fmt.Sprintf("/v1/sys/plugins/catalog/%s", name) + } + + return path +} diff --git a/vendor/github.com/hashicorp/vault/api/sys_policy.go b/vendor/github.com/hashicorp/vault/api/sys_policy.go index 1fa32597efb..c0c239f960c 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_policy.go +++ b/vendor/github.com/hashicorp/vault/api/sys_policy.go @@ -9,7 +9,11 @@ import ( ) func (c *Sys) ListPolicies() ([]string, error) { - r := c.c.NewRequest("GET", "/v1/sys/policy") + r := c.c.NewRequest("LIST", "/v1/sys/policies/acl") + // Set this for broader compatibility, but we use LIST above to be able to + // handle the wrapping lookup function + r.Method = "GET" + r.Params.Set("list", "true") ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() @@ -28,7 +32,7 @@ func (c *Sys) ListPolicies() ([]string, error) { } var result []string - err = mapstructure.Decode(secret.Data["policies"], &result) + err = mapstructure.Decode(secret.Data["keys"], &result) if err != nil { return nil, err } @@ -69,10 +73,10 @@ func (c *Sys) GetPolicy(name string) (string, error) { func (c *Sys) PutPolicy(name, rules string) error { body := map[string]string{ - "rules": rules, + "policy": rules, } - r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/policy/%s", name)) + r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/policies/acl/%s", name)) if err := r.SetJSONBody(body); err != nil { return err } @@ -89,7 +93,7 @@ func (c *Sys) PutPolicy(name, rules string) error { } func (c *Sys) DeletePolicy(name string) error { - r := c.c.NewRequest("DELETE", fmt.Sprintf("/v1/sys/policy/%s", name)) + r := c.c.NewRequest("DELETE", fmt.Sprintf("/v1/sys/policies/acl/%s", name)) ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() diff --git a/vendor/github.com/hashicorp/vault/api/sys_seal.go b/vendor/github.com/hashicorp/vault/api/sys_seal.go index 7cc32ac33c8..301d3f26a10 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_seal.go +++ b/vendor/github.com/hashicorp/vault/api/sys_seal.go @@ -41,6 +41,15 @@ func (c *Sys) Unseal(shard string) (*SealStatusResponse, error) { return sealStatusRequest(c, r) } +func (c *Sys) UnsealWithOptions(opts *UnsealOpts) (*SealStatusResponse, error) { + r := c.c.NewRequest("PUT", "/v1/sys/unseal") + if err := r.SetJSONBody(opts); err != nil { + return nil, err + } + + return sealStatusRequest(c, r) +} + func sealStatusRequest(c *Sys, r *Request) (*SealStatusResponse, error) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() @@ -57,13 +66,21 @@ func sealStatusRequest(c *Sys, r *Request) (*SealStatusResponse, error) { type SealStatusResponse struct { Type string `json:"type"` + Initialized bool `json:"initialized"` Sealed bool `json:"sealed"` T int `json:"t"` N int `json:"n"` Progress int `json:"progress"` Nonce string `json:"nonce"` Version string `json:"version"` + Migration bool `json:"migration"` ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` RecoverySeal bool `json:"recovery_seal"` } + +type UnsealOpts struct { + Key string `json:"key"` + Reset bool `json:"reset"` + Migrate bool `json:"migrate"` +} diff --git a/vendor/github.com/hashicorp/vault/helper/consts/plugin_types.go b/vendor/github.com/hashicorp/vault/helper/consts/plugin_types.go new file mode 100644 index 00000000000..e0a00e4860c --- /dev/null +++ b/vendor/github.com/hashicorp/vault/helper/consts/plugin_types.go @@ -0,0 +1,59 @@ +package consts + +import "fmt" + +var PluginTypes = []PluginType{ + PluginTypeUnknown, + PluginTypeCredential, + PluginTypeDatabase, + PluginTypeSecrets, +} + +type PluginType uint32 + +// This is a list of PluginTypes used by Vault. +// If we need to add any in the future, it would +// be best to add them to the _end_ of the list below +// because they resolve to incrementing numbers, +// which may be saved in state somewhere. Thus if +// the name for one of those numbers changed because +// a value were added to the middle, that could cause +// the wrong plugin types to be read from storage +// for a given underlying number. Example of the problem +// here: https://play.golang.org/p/YAaPw5ww3er +const ( + PluginTypeUnknown PluginType = iota + PluginTypeCredential + PluginTypeDatabase + PluginTypeSecrets +) + +func (p PluginType) String() string { + switch p { + case PluginTypeUnknown: + return "unknown" + case PluginTypeCredential: + return "auth" + case PluginTypeDatabase: + return "database" + case PluginTypeSecrets: + return "secret" + default: + return "unsupported" + } +} + +func ParsePluginType(pluginType string) (PluginType, error) { + switch pluginType { + case "unknown": + return PluginTypeUnknown, nil + case "auth": + return PluginTypeCredential, nil + case "database": + return PluginTypeDatabase, nil + case "secret": + return PluginTypeSecrets, nil + default: + return PluginTypeUnknown, fmt.Errorf("%q is not a supported plugin type", pluginType) + } +} diff --git a/vendor/github.com/hashicorp/vault/helper/consts/replication.go b/vendor/github.com/hashicorp/vault/helper/consts/replication.go index bdad1552257..a7e0edea1c5 100644 --- a/vendor/github.com/hashicorp/vault/helper/consts/replication.go +++ b/vendor/github.com/hashicorp/vault/helper/consts/replication.go @@ -16,7 +16,7 @@ const ( // ensure no overlap between old and new values. ReplicationUnknown ReplicationState = 0 - ReplicationPerformancePrimary ReplicationState = 1 << iota + ReplicationPerformancePrimary ReplicationState = 1 << iota // Note -- iota is 5 here! ReplicationPerformanceSecondary OldSplitReplicationBootstrapping ReplicationDRPrimary @@ -51,6 +51,39 @@ func (r ReplicationState) string() string { return "unknown" } +func (r ReplicationState) StateStrings() []string { + var ret []string + if r.HasState(ReplicationPerformanceSecondary) { + ret = append(ret, "perf-secondary") + } + if r.HasState(ReplicationPerformancePrimary) { + ret = append(ret, "perf-primary") + } + if r.HasState(ReplicationPerformanceBootstrapping) { + ret = append(ret, "perf-bootstrapping") + } + if r.HasState(ReplicationPerformanceDisabled) { + ret = append(ret, "perf-disabled") + } + if r.HasState(ReplicationDRPrimary) { + ret = append(ret, "dr-primary") + } + if r.HasState(ReplicationDRSecondary) { + ret = append(ret, "dr-secondary") + } + if r.HasState(ReplicationDRBootstrapping) { + ret = append(ret, "dr-bootstrapping") + } + if r.HasState(ReplicationDRDisabled) { + ret = append(ret, "dr-disabled") + } + if r.HasState(ReplicationPerformanceStandby) { + ret = append(ret, "perfstandby") + } + + return ret +} + func (r ReplicationState) GetDRString() string { switch { case r.HasState(ReplicationDRBootstrapping): diff --git a/vendor/vendor.json b/vendor/vendor.json index f8f968d037d..cdda022b694 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -54,10 +54,7 @@ {"path":"github.com/bgentry/speakeasy/example","checksumSHA1":"twtRfb6484vfr2qqjiFkLThTjcQ=","revision":"36e9cfdd690967f4f690c6edcc9ffacd006014a0"}, {"path":"github.com/boltdb/bolt","checksumSHA1":"R1Q34Pfnt197F/nCOO9kG8c+Z90=","comment":"v1.2.0","revision":"2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8","revisionTime":"2017-07-17T17:11:48Z","version":"v1.3.1","versionExact":"v1.3.1"}, {"path":"github.com/burntsushi/toml","checksumSHA1":"InIrfOI7Ys1QqZpCgTB4yW1G32M=","revision":"99064174e013895bbd9b025c31100bd1d9b590ca","revisionTime":"2016-07-17T15:07:09Z"}, - {"path":"github.com/circonus-labs/circonus-gometrics","checksumSHA1":"/qvtQq5y0RZCsRyOOsan87V2AL0=","revision":"dd698dc110872f6e554abf74a7740fc363354086","revisionTime":"2018-08-20T20:09:38Z"}, - {"path":"github.com/circonus-labs/circonus-gometrics/api","checksumSHA1":"Lll5SHEsVto8Eqbrj7NVj7BfgDI=","revision":"dd698dc110872f6e554abf74a7740fc363354086","revisionTime":"2018-08-20T20:09:38Z"}, - {"path":"github.com/circonus-labs/circonus-gometrics/api/config","checksumSHA1":"bQhz/fcyZPmuHSH2qwC4ZtATy5c=","revision":"dd698dc110872f6e554abf74a7740fc363354086","revisionTime":"2018-08-20T20:09:38Z"}, - {"path":"github.com/circonus-labs/circonus-gometrics/checkmgr","checksumSHA1":"Ij8yB33E0Kk+GfTkNRoF1mG26dc=","revision":"dd698dc110872f6e554abf74a7740fc363354086","revisionTime":"2018-08-20T20:09:38Z"}, + {"path":"github.com/circonus-labs/circonus-gometrics","checksumSHA1":"2nmREcVPEmfsKz/OFMSWI4g4iDw=","revision":"03033123293bfe2241d1f7e1a3de8fb508265251","revisionTime":"2018-09-28T18:16:22Z","tree":true}, {"path":"github.com/circonus-labs/circonusllhist","checksumSHA1":"VbfeVqeOM+dTNxCmpvmYS0LwQn0=","revision":"7d649b46cdc2cd2ed102d350688a75a4fd7778c6","revisionTime":"2016-11-21T13:51:53Z"}, {"path":"github.com/containerd/console","checksumSHA1":"IGtuR58l2zmYRcNf8sPDlCSgovE=","origin":"github.com/opencontainers/runc/vendor/github.com/containerd/console","revision":"459bfaec1fc6c17d8bfb12d0a0f69e7e7271ed2a","revisionTime":"2018-08-23T14:46:37Z"}, {"path":"github.com/containerd/continuity/pathdriver","checksumSHA1":"GqIrOttKaO7k6HIaHQLPr3cY7rY=","origin":"github.com/docker/docker/vendor/github.com/containerd/continuity/pathdriver","revision":"320063a2ad06a1d8ada61c94c29dbe44e2d87473","revisionTime":"2018-08-16T08:14:46Z"}, @@ -153,18 +150,19 @@ {"path":"github.com/gorhill/cronexpr/cronexpr","checksumSHA1":"Nd/7mZb0T6Gj6+AymyOPsNCQSJs=","comment":"1.0.0","revision":"a557574d6c024ed6e36acc8b610f5f211c91568a"}, {"path":"github.com/gorilla/context","checksumSHA1":"g/V4qrXjUGG9B+e3hB+4NAYJ5Gs=","revision":"08b5f424b9271eedf6f9f0ce86cb9396ed337a42","revisionTime":"2016-08-17T18:46:32Z"}, {"path":"github.com/gorilla/mux","checksumSHA1":"STQSdSj2FcpCf0NLfdsKhNutQT0=","revision":"e48e440e4c92e3251d812f8ce7858944dfa3331c","revisionTime":"2018-08-07T07:52:56Z"}, - {"path":"github.com/hashicorp/consul-template","checksumSHA1":"JGDXrGETgjJYNGIb//si5C6JIj4=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/child","checksumSHA1":"AhDPiKa7wzh3SE6Gx0WrsDYwBHg=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/config","checksumSHA1":"V+1cP51VHrIsoayaMrKyMfAjKQk=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/dependency","checksumSHA1":"ooC1P0Z8MTQ+JYc2cxTia+6w41w=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/logging","checksumSHA1":"o5N7SV389Ej+3b1iRNmz1dx5e1M=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/manager","checksumSHA1":"Qf3HTBNa6NpM2h/aecUNmpXA6eo=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/signals","checksumSHA1":"YSEUV/9/k85XciRKu0cngxdjZLE=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/template","checksumSHA1":"0mSanQgyqUc3X44C7IobVyy8JJc=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/version","checksumSHA1":"ZEI6EWoUxsaOnaajcxxqH7cnIH4=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, - {"path":"github.com/hashicorp/consul-template/watch","checksumSHA1":"wLwStBhxVRf0qaE5fIN4yWuBkB4=","revision":"f8c8205caf458dfd0ecab69d029ab112803aa587","revisionTime":"2018-06-12T16:16:25Z"}, + {"path":"github.com/hashicorp/consul-template","checksumSHA1":"+AGSqY+9kpGX5rrQDBWpgzaDKSA=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/child","checksumSHA1":"AhDPiKa7wzh3SE6Gx0WrsDYwBHg=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/config","checksumSHA1":"0vr6paBMXD7ZYSmtfJpjfjZJKic=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/dependency","checksumSHA1":"PYz8+xNjJkv+pyqo/d5f2qVSH+w=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/logging","checksumSHA1":"o5N7SV389Ej+3b1iRNmz1dx5e1M=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/manager","checksumSHA1":"FXkwzbFD6/LTHyf8bjpgxGvFXOE=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/renderer","checksumSHA1":"DUHtghMoLyrgPhv4lexVniBuWYk=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/signals","checksumSHA1":"YSEUV/9/k85XciRKu0cngxdjZLE=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/template","checksumSHA1":"Y0Ws3O64np8sFDE/3vAx8lFUHxc=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/version","checksumSHA1":"RbpOltpZ0PLi1NA6Senz9sWFRSc=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, + {"path":"github.com/hashicorp/consul-template/watch","checksumSHA1":"cJxopvJKg7DBBb8tnDsfmBp5Q8I=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, {"path":"github.com/hashicorp/consul/agent/consul/autopilot","checksumSHA1":"+I7fgoQlrnTUGW5krqNLadWwtjg=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, - {"path":"github.com/hashicorp/consul/api","checksumSHA1":"7UvyPiYTxcB8xqRlULAT3X8+8zE=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, + {"path":"github.com/hashicorp/consul/api","checksumSHA1":"sjEf6EMTPP4NT3m5a0JJmlbLZ8Y=","revision":"39f93f011e591c842acc8053a7f5972aa6e592fd","revisionTime":"2018-07-12T16:33:56Z"}, {"path":"github.com/hashicorp/consul/command/flags","checksumSHA1":"soNN4xaHTbeXFgNkZ7cX0gbFXQk=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, {"path":"github.com/hashicorp/consul/lib","checksumSHA1":"Nrh9BhiivRyJiuPzttstmq9xl/w=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, {"path":"github.com/hashicorp/consul/lib/freeport","checksumSHA1":"E28E4zR1FN2v1Xiq4FUER7KVN9M=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, @@ -194,7 +192,7 @@ {"path":"github.com/hashicorp/go-multierror","checksumSHA1":"lrSl49G23l6NhfilxPM0XFs5rZo=","revision":"d30f09973e19c1dfcd120b2d9c4f168e68d6b5d5"}, {"path":"github.com/hashicorp/go-plugin","checksumSHA1":"IFwYSAmxxM+fV0w9LwdWKqFOCgg=","revision":"f444068e8f5a19853177f7aa0aea7e7d95b5b528","revisionTime":"2018-12-12T15:08:38Z"}, {"path":"github.com/hashicorp/go-plugin/internal/proto","checksumSHA1":"Ikbb1FngsPR79bHhr2UmKk4CblI=","revision":"f444068e8f5a19853177f7aa0aea7e7d95b5b528","revisionTime":"2018-12-12T15:08:38Z"}, - {"path":"github.com/hashicorp/go-retryablehttp","checksumSHA1":"/yKfFSspjuDzyOe/bBslrPzyfYM=","revision":"e651d75abec6fbd4f2c09508f72ae7af8a8b7171","revisionTime":"2018-07-18T19:50:05Z"}, + {"path":"github.com/hashicorp/go-retryablehttp","checksumSHA1":"9SqwC2BzFbsWulQuBG2+QEliTpo=","revision":"73489d0a1476f0c9e6fb03f9c39241523a496dfd","revisionTime":"2019-01-26T20:33:39Z"}, {"path":"github.com/hashicorp/go-rootcerts","checksumSHA1":"A1PcINvF3UiwHRKn8UcgARgvGRs=","revision":"6bb64b370b90e7ef1fa532be9e591a81c3493e00","revisionTime":"2016-05-03T14:34:40Z"}, {"path":"github.com/hashicorp/go-safetemp","checksumSHA1":"CduvzBFfTv77nhjtXPGdIjQQLMI=","revision":"b1a1dbde6fdc11e3ae79efd9039009e22d4ae240","revisionTime":"2018-03-26T21:11:50Z"}, {"path":"github.com/hashicorp/go-sockaddr","checksumSHA1":"J47ySO1q0gcnmoMnir1q1loKzCk=","revision":"6d291a969b86c4b633730bfc6b8b9d64c3aafed9","revisionTime":"2018-03-20T11:50:54Z"}, @@ -227,9 +225,9 @@ {"path":"github.com/hashicorp/serf","checksumSHA1":"9omt7lEuhBNSdgT32ThEzXn/aTU=","revision":"c7f3bc96b40972e67dfbe007c1fa825cf59ac8c2","revisionTime":"2019-01-04T15:39:47Z"}, {"path":"github.com/hashicorp/serf/coordinate","checksumSHA1":"0PeWsO2aI+2PgVYlYlDPKfzCLEQ=","revision":"c7f3bc96b40972e67dfbe007c1fa825cf59ac8c2","revisionTime":"2019-01-04T15:39:47Z"}, {"path":"github.com/hashicorp/serf/serf","checksumSHA1":"siLn7zwVHQk070rpd99BTktGfTs=","revision":"c7f3bc96b40972e67dfbe007c1fa825cf59ac8c2","revisionTime":"2019-01-04T15:39:47Z"}, - {"path":"github.com/hashicorp/vault/api","checksumSHA1":"DP7dd8OErZVF0q+XfPo0RGkDcLk=","revision":"6e8d91a59c34bd9f323397c30be9651422295c65","revisionTime":"2018-09-19T17:09:49Z"}, + {"path":"github.com/hashicorp/vault/api","checksumSHA1":"laYVVZeqao4WdIS/MKd0be7Z3yA=","revision":"85909e3373aa743c34a6a0ab59131f61fd9e8e43","revisionTime":"2019-02-12T14:05:52Z"}, {"path":"github.com/hashicorp/vault/helper/compressutil","checksumSHA1":"bSdPFOHaTwEvM4PIvn0PZfn75jM=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, - {"path":"github.com/hashicorp/vault/helper/consts","checksumSHA1":"QNGGvSYtwk6VCkj4laZPjM2301E=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, + {"path":"github.com/hashicorp/vault/helper/consts","checksumSHA1":"3mRK/pBPUH8rIPrILmuJWynWQVo=","revision":"85909e3373aa743c34a6a0ab59131f61fd9e8e43","revisionTime":"2019-02-12T14:05:52Z"}, {"path":"github.com/hashicorp/vault/helper/hclutil","checksumSHA1":"RlqPBLOexQ0jj6jomhiompWKaUg=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, {"path":"github.com/hashicorp/vault/helper/jsonutil","checksumSHA1":"POgkM3GrjRFw6H3sw95YNEs552A=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, {"path":"github.com/hashicorp/vault/helper/parseutil","checksumSHA1":"HA2MV/2XI0HcoThSRxQCaBZR2ps=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, From da7a0dc70bbc5a862438eef51dc4639d0d886dbd Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Thu, 4 Apr 2019 15:27:55 +0000 Subject: [PATCH 04/13] taskrunner: pass configured Vault namespace into TaskTemplateConfig --- .../taskrunner/template/template.go | 3 ++ .../taskrunner/template/template_test.go | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/client/allocrunner/taskrunner/template/template.go b/client/allocrunner/taskrunner/template/template.go index 7421b531894..e029037d064 100644 --- a/client/allocrunner/taskrunner/template/template.go +++ b/client/allocrunner/taskrunner/template/template.go @@ -645,6 +645,9 @@ func newRunnerConfig(config *TaskTemplateManagerConfig, conf.Vault.Address = &cc.VaultConfig.Addr conf.Vault.Token = &config.VaultToken conf.Vault.Grace = helper.TimeToPtr(vaultGrace) + if config.ClientConfig.VaultConfig.Namespace != "" { + conf.Vault.Namespace = &config.ClientConfig.VaultConfig.Namespace + } if strings.HasPrefix(cc.VaultConfig.Addr, "https") || cc.VaultConfig.TLSCertFile != "" { skipVerify := cc.VaultConfig.TLSSkipVerify != nil && *cc.VaultConfig.TLSSkipVerify diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index b736d436676..ac2b9f5f2e8 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1326,6 +1326,54 @@ func TestTaskTemplateManager_Config_VaultGrace(t *testing.T) { assert.Equal(10*time.Second, *ctconf.Vault.Grace, "Vault Grace Value") } +// TestTaskTemplateManager_Config_VaultNamespace asserts the Vault namespace setting is +// propagated to consul-template's configuration. +func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) { + t.Parallel() + assert := assert.New(t) + + testNS := "test-namespace" + c := config.DefaultConfig() + c.Node = mock.Node() + c.VaultConfig = &sconfig.VaultConfig{ + Enabled: helper.BoolToPtr(true), + Addr: "https://localhost/", + TLSServerName: "notlocalhost", + Namespace: testNS, + } + + alloc := mock.Alloc() + config := &TaskTemplateManagerConfig{ + ClientConfig: c, + VaultToken: "token", + + // Make a template that will render immediately + Templates: []*structs.Template{ + { + EmbeddedTmpl: "bar", + DestPath: "foo", + ChangeMode: structs.TemplateChangeModeNoop, + VaultGrace: 10 * time.Second, + }, + { + EmbeddedTmpl: "baz", + DestPath: "bam", + ChangeMode: structs.TemplateChangeModeNoop, + VaultGrace: 100 * time.Second, + }, + }, + EnvBuilder: taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region), + } + + ctmplMapping, err := parseTemplateConfigs(config) + assert.Nil(err, "Parsing Templates") + + ctconf, err := newRunnerConfig(config, ctmplMapping) + assert.Nil(err, "Building Runner Config") + assert.NotNil(ctconf.Vault.Grace, "Vault Grace Pointer") + assert.Equal(testNS, *ctconf.Vault.Namespace, "Vault Namespace Value") +} + func TestTaskTemplateManager_BlockedEvents(t *testing.T) { t.Parallel() require := require.New(t) From d6b3ac7f31d8531459bb90536a806f0fb31ce127 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Thu, 4 Apr 2019 21:02:28 +0000 Subject: [PATCH 05/13] client: gofmt --- client/allocrunner/taskrunner/template/template_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index ac2b9f5f2e8..803162e72e9 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1339,13 +1339,13 @@ func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) { Enabled: helper.BoolToPtr(true), Addr: "https://localhost/", TLSServerName: "notlocalhost", - Namespace: testNS, + Namespace: testNS, } alloc := mock.Alloc() config := &TaskTemplateManagerConfig{ - ClientConfig: c, - VaultToken: "token", + ClientConfig: c, + VaultToken: "token", // Make a template that will render immediately Templates: []*structs.Template{ From 473b6d041b6ea4e2dac0536c02fe42c195dc9072 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 01:16:51 +0000 Subject: [PATCH 06/13] docs: -vault-namespace, VAULT_NAMESPACE, and config agent: added VAULT_NAMESPACE env-based configuration --- client/vaultclient/vaultclient.go | 1 + command/agent/command.go | 7 +++++++ nomad/vault.go | 2 +- website/source/docs/commands/agent.html.md.erb | 3 +++ .../guides/operations/vault-integration/index.html.md | 10 +++++++--- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/vaultclient/vaultclient.go b/client/vaultclient/vaultclient.go index 5512882e977..5bebbd883e0 100644 --- a/client/vaultclient/vaultclient.go +++ b/client/vaultclient/vaultclient.go @@ -165,6 +165,7 @@ func NewVaultClient(config *config.VaultConfig, logger hclog.Logger, tokenDerive // SetHeaders above will replace all headers, make this call second if config.Namespace != "" { + logger.Debug("configuring Vault namespace", "namespace", config.Namespace) client.SetNamespace(config.Namespace) } diff --git a/command/agent/command.go b/command/agent/command.go index 1ac133e5345..d30958a5b5b 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -261,6 +261,13 @@ func (c *Command) readConfig() *Config { } } + // Check to see if we should read the Vault namespace from the environment + if config.Vault.Namespace == "" { + if ns, ok := os.LookupEnv("VAULT_NAMESPACE"); ok { + config.Vault.Namespace = ns + } + } + // Default the plugin directory to be under that of the data directory if it // isn't explicitly specified. if config.PluginDir == "" && config.DataDir != "" { diff --git a/nomad/vault.go b/nomad/vault.go index f2915345b4d..ed71c5e4e9d 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -254,7 +254,7 @@ func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVault } if c.Namespace != "" { - logger.Debug("Setting Vault namespace", "namespace", c.Namespace) + logger.Debug("configuring Vault namespace", "namespace", c.Namespace) v.client.SetNamespace(c.Namespace) } diff --git a/website/source/docs/commands/agent.html.md.erb b/website/source/docs/commands/agent.html.md.erb index 474ec7f1490..f2a25197543 100644 --- a/website/source/docs/commands/agent.html.md.erb +++ b/website/source/docs/commands/agent.html.md.erb @@ -101,6 +101,9 @@ via CLI arguments. The `agent` command accepts the following arguments: integration. * `vault-cert-file=`: The path to the certificate for Vault communication. * `vault-key-file=`: The path to the private key for Vault communication. +* `vault-namespace=`: The Vault namespace used for the integration. + Required for servers and clients. Overrides the Vault namespace read from the + VAULT_NAMESPACE environment variable. * `vault-tls-skip-verify`: A boolean that determines whether to skip SSL certificate verification. * `vault-tls-server-name=`: Used to set the SNI host when connecting to diff --git a/website/source/guides/operations/vault-integration/index.html.md b/website/source/guides/operations/vault-integration/index.html.md index c6783018ebc..160eeef6662 100644 --- a/website/source/guides/operations/vault-integration/index.html.md +++ b/website/source/guides/operations/vault-integration/index.html.md @@ -261,9 +261,12 @@ Nomad Server's configuration file located at `/etc/nomad.d/nomad.hcl`. Provide the token you generated in the previous step in the `vault` stanza of your Nomad server configuration. The token can also be provided as an environment variable called `VAULT_TOKEN`. Be sure to specify the `nomad-cluster-role` in the -[create_from_role][create-from-role] option. After following these steps and -enabling Vault, the `vault` stanza in your Nomad server configuration will be -similar to what is shown below: +[create_from_role][create-from-role] option. If using +[Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces/index.html), +modify both the client and server configuration to include the namespace; +alternatively, it can be provided in the environment variable `VAULT_NAMESPACE`. +After following these steps and enabling Vault, the `vault` stanza in your Nomad +server configuration will be similar to what is shown below: ```hcl vault { @@ -272,6 +275,7 @@ vault { task_token_ttl = "1h" create_from_role = "nomad-cluster" token = "" + namespace = "" } ``` From 8d9f2c26b561ea1945ce2fc750c995876bc4f159 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 04:21:13 +0000 Subject: [PATCH 07/13] e2e/vault: updated e2e vault tests to use version-specific policy creation endpoint (old servers are not compatible with new client) --- e2e/vault/matrix_test.go | 4 ++++ e2e/vault/vault_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/e2e/vault/matrix_test.go b/e2e/vault/matrix_test.go index 1dc786a6ae3..86dcf96f685 100644 --- a/e2e/vault/matrix_test.go +++ b/e2e/vault/matrix_test.go @@ -3,6 +3,10 @@ package vault var ( // versions is the set of Vault versions we test for backwards compatibility versions = []string{ + "1.1.0", + "1.0.3", + "1.0.2", + "1.0.1", "1.0.0", "0.11.5", "0.11.4", diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index 2fcdb9b638b..da1dc9f4e86 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -12,9 +12,12 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" + "github.com/hashicorp/go-version" + "github.com/hashicorp/nomad/command/agent" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs/config" @@ -254,8 +257,32 @@ func testVaultCompatibility(t *testing.T, vault string) { func setupVault(t *testing.T, client *vapi.Client) string { // Write the policy sys := client.Sys() - if err := sys.PutPolicy("nomad-server", policy); err != nil { - t.Fatalf("failed to create policy: %v", err) + + // pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint + // perform this using a raw HTTP request + newApi, err := version.NewVersion("0.9.0") + if err != nil { + t.Fatalf("failed to parse comparison version: %v", err) + } + testVersion, err := version.NewVersion(strings.TrimPrefix(t.Name(), "TestVaultCompatibility/")) + if err != nil { + t.Fatalf("failed to parse test version from '%v': %v", t.Name(), err) + } + if testVersion.LessThan(newApi) { + body := map[string]string{ + "rules": policy, + } + request := client.NewRequest("PUT", fmt.Sprintf("/v1/sys/policy/%s", "nomad-server")) + if err := request.SetJSONBody(body); err != nil { + t.Fatalf("failed to set JSON body on legacy policy creation: %v", err) + } + if _, err := client.RawRequest(request); err != nil { + t.Fatalf("failed to create legacy policy: %v", err) + } + } else { + if err := sys.PutPolicy("nomad-server", policy); err != nil { + t.Fatalf("failed to create policy: %v", err) + } } // Build the role From 87400ea0d27fb2f4c906d2d77725c2698fd7b615 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 13:05:54 +0000 Subject: [PATCH 08/13] taskrunner: removed some unecessary config from a test --- .../taskrunner/template/template_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index 803162e72e9..b8ce44a7d39 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1346,22 +1346,6 @@ func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) { config := &TaskTemplateManagerConfig{ ClientConfig: c, VaultToken: "token", - - // Make a template that will render immediately - Templates: []*structs.Template{ - { - EmbeddedTmpl: "bar", - DestPath: "foo", - ChangeMode: structs.TemplateChangeModeNoop, - VaultGrace: 10 * time.Second, - }, - { - EmbeddedTmpl: "baz", - DestPath: "bam", - ChangeMode: structs.TemplateChangeModeNoop, - VaultGrace: 100 * time.Second, - }, - }, EnvBuilder: taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region), } From 99ce9859d432fe6bbc1199f27d8fb95980a3c195 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 13:21:54 +0000 Subject: [PATCH 09/13] vault e2e: pass vault version into setup instead of having to infer it from test name --- .../taskrunner/template/template_test.go | 2 +- e2e/vault/vault_test.go | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index b8ce44a7d39..a0d6484a6ad 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1346,7 +1346,7 @@ func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) { config := &TaskTemplateManagerConfig{ ClientConfig: c, VaultToken: "token", - EnvBuilder: taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region), + EnvBuilder: taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region), } ctmplMapping, err := parseTemplateConfigs(config) diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index da1dc9f4e86..e985bcfb336 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "testing" "time" @@ -187,20 +186,20 @@ func TestVaultCompatibility(t *testing.T) { for version, vaultBin := range vaultBinaries { vbin := vaultBin t.Run(version, func(t *testing.T) { - testVaultCompatibility(t, vbin) + testVaultCompatibility(t, vbin, version) }) } } // testVaultCompatibility tests compatibility with the given vault binary -func testVaultCompatibility(t *testing.T, vault string) { +func testVaultCompatibility(t *testing.T, vault string, version string) { require := require.New(t) // Create a Vault server v := testutil.NewTestVaultFromPath(t, vault) defer v.Stop() - token := setupVault(t, v.Client) + token := setupVault(t, v.Client, version) // Create a Nomad agent using the created vault nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) { @@ -254,17 +253,14 @@ func testVaultCompatibility(t *testing.T, vault string) { // setupVault takes the Vault client and creates the required policies and // roles. It returns the token that should be used by Nomad -func setupVault(t *testing.T, client *vapi.Client) string { +func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string { // Write the policy sys := client.Sys() // pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint // perform this using a raw HTTP request - newApi, err := version.NewVersion("0.9.0") - if err != nil { - t.Fatalf("failed to parse comparison version: %v", err) - } - testVersion, err := version.NewVersion(strings.TrimPrefix(t.Name(), "TestVaultCompatibility/")) + newApi, _ := version.NewVersion("0.9.0") + testVersion, err := version.NewVersion(vaultVersion) if err != nil { t.Fatalf("failed to parse test version from '%v': %v", t.Name(), err) } From ce5ca22a681b80b6c61d5be49a47b27139f3257e Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 5 Apr 2019 11:01:14 -0400 Subject: [PATCH 10/13] Update nomad/structs/config/vault.go Co-Authored-By: cgbaker --- nomad/structs/config/vault.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index c568a70ecfb..3a0b8fab632 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -37,7 +37,7 @@ type VaultConfig struct { // role the token is from. Role string `mapstructure:"create_from_role"` - // Namespaces sets the Vault namespace used for all calls against the + // Namespace sets the Vault namespace used for all calls against the // Vault API. If this is unset, then Nomad does not use Vault namespaces. Namespace string `mapstructure:"namespace"` From 4c729e6b497400a6ff6fffb58e8ce30a400f3ebf Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 17:07:40 +0000 Subject: [PATCH 11/13] vault client test: minor formatting vendor: using upstream circonus-gometrics --- client/vaultclient/vaultclient_test.go | 5 +--- .../circonus-gometrics/CHANGELOG.md | 9 ++++++ .../circonus-gometrics/Gopkg.lock | 18 ++++++------ .../circonus-gometrics/Gopkg.toml | 28 ++----------------- .../circonus-gometrics/submit.go | 4 +-- vendor/vendor.json | 5 +++- 6 files changed, 28 insertions(+), 41 deletions(-) diff --git a/client/vaultclient/vaultclient_test.go b/client/vaultclient/vaultclient_test.go index 6dad3778e8e..f3f6a879c6d 100644 --- a/client/vaultclient/vaultclient_test.go +++ b/client/vaultclient/vaultclient_test.go @@ -108,10 +108,7 @@ func TestVaultClient_NamespaceSupport(t *testing.T) { conf.VaultConfig.Token = "testvaulttoken" conf.VaultConfig.Namespace = testNs c, err := NewVaultClient(conf.VaultConfig, logger, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - + require.NoError(err) require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) } diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md b/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md index 4a0ab2c0d80..8aa09ea0e04 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md +++ b/vendor/github.com/circonus-labs/circonus-gometrics/CHANGELOG.md @@ -1,3 +1,12 @@ +# v2.2.6 + +* fix: func signature to match go-retryablehttp update +* upd: dependency go-retryablehttp, lock to v0.5.2 to prevent future breaking patch features + +# v2.2.5 + +* upd: switch from tracking master to versions for retryablehttp and circonusllhist now that both repositories are doing releases + # v2.2.4 * fix: worksheet.graphs is a required attribute. worksheet.smart_queries is an optional attribute. diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.lock b/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.lock index 0f993c9a5e7..d306f40116a 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.lock +++ b/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.lock @@ -2,28 +2,28 @@ [[projects]] - branch = "master" name = "github.com/circonus-labs/circonusllhist" packages = ["."] - revision = "5eb751da55c6d3091faf3861ec5062ae91fee9d0" + revision = "87d4d00b35adeefe4911ece727838749e0fab113" + version = "v0.1.3" [[projects]] - branch = "master" name = "github.com/hashicorp/go-cleanhttp" packages = ["."] - revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" + revision = "e8ab9daed8d1ddd2d3c4efba338fe2eeae2e4f18" + version = "v0.5.0" [[projects]] - branch = "master" name = "github.com/hashicorp/go-retryablehttp" packages = ["."] - revision = "e651d75abec6fbd4f2c09508f72ae7af8a8b7171" + revision = "73489d0a1476f0c9e6fb03f9c39241523a496dfd" + version = "v0.5.2" [[projects]] name = "github.com/pkg/errors" packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" [[projects]] branch = "master" @@ -34,6 +34,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6db34ba31cd011426f28b5db0dbe259c4dc3787fb2074b2c06cb382385a90242" + inputs-digest = "ff81639f2f1513555846304ee903af4d13a0f0f181e140e1ebb1d71aa18fb5fb" solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.toml b/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.toml index fa41a53c0a6..bb40a91e236 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.toml +++ b/vendor/github.com/circonus-labs/circonus-gometrics/Gopkg.toml @@ -1,36 +1,14 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - [[constraint]] - branch = "master" name = "github.com/circonus-labs/circonusllhist" + version = "0.1.3" [[constraint]] - branch = "master" name = "github.com/hashicorp/go-retryablehttp" + version = "=0.5.2" [[constraint]] name = "github.com/pkg/errors" - version = "0.8.0" + version = "0.8.1" [[constraint]] branch = "master" diff --git a/vendor/github.com/circonus-labs/circonus-gometrics/submit.go b/vendor/github.com/circonus-labs/circonus-gometrics/submit.go index 7af5adb3366..f99bc4ced9d 100644 --- a/vendor/github.com/circonus-labs/circonus-gometrics/submit.go +++ b/vendor/github.com/circonus-labs/circonus-gometrics/submit.go @@ -17,7 +17,7 @@ import ( "time" "github.com/circonus-labs/circonus-gometrics/api" - "github.com/hashicorp/go-retryablehttp" + retryablehttp "github.com/hashicorp/go-retryablehttp" "github.com/pkg/errors" ) @@ -141,8 +141,8 @@ func (m *CirconusMetrics) trapCall(payload []byte) (int, error) { client.CheckRetry = retryPolicy attempts := -1 - client.RequestLogHook = func(logger retryablehttp.Logger, req *http.Request, retryNumber int) { + //client.RequestLogHook = func(logger *log.Logger, req *http.Request, retryNumber int) { attempts = retryNumber } diff --git a/vendor/vendor.json b/vendor/vendor.json index cdda022b694..1a44b6fb16b 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -54,7 +54,10 @@ {"path":"github.com/bgentry/speakeasy/example","checksumSHA1":"twtRfb6484vfr2qqjiFkLThTjcQ=","revision":"36e9cfdd690967f4f690c6edcc9ffacd006014a0"}, {"path":"github.com/boltdb/bolt","checksumSHA1":"R1Q34Pfnt197F/nCOO9kG8c+Z90=","comment":"v1.2.0","revision":"2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8","revisionTime":"2017-07-17T17:11:48Z","version":"v1.3.1","versionExact":"v1.3.1"}, {"path":"github.com/burntsushi/toml","checksumSHA1":"InIrfOI7Ys1QqZpCgTB4yW1G32M=","revision":"99064174e013895bbd9b025c31100bd1d9b590ca","revisionTime":"2016-07-17T15:07:09Z"}, - {"path":"github.com/circonus-labs/circonus-gometrics","checksumSHA1":"2nmREcVPEmfsKz/OFMSWI4g4iDw=","revision":"03033123293bfe2241d1f7e1a3de8fb508265251","revisionTime":"2018-09-28T18:16:22Z","tree":true}, + {"path":"github.com/circonus-labs/circonus-gometrics","checksumSHA1":"H4RhrnI0P34qLB9345G4r7CAwpU=","revision":"d6e3aea90ab9f90fe8456e13fc520f43d102da4d","revisionTime":"2019-01-28T15:50:09Z","version":"=v2","versionExact":"v2"}, + {"path":"github.com/circonus-labs/circonus-gometrics/api","checksumSHA1":"xtzLG2UjYF1lnD33Wk+Nu/KOO6E=","revision":"d6e3aea90ab9f90fe8456e13fc520f43d102da4d","revisionTime":"2019-01-28T15:50:09Z","version":"=v2","versionExact":"v2"}, + {"path":"github.com/circonus-labs/circonus-gometrics/api/config","checksumSHA1":"bQhz/fcyZPmuHSH2qwC4ZtATy5c=","revision":"d6e3aea90ab9f90fe8456e13fc520f43d102da4d","revisionTime":"2019-01-28T15:50:09Z","version":"=v2","versionExact":"v2"}, + {"path":"github.com/circonus-labs/circonus-gometrics/checkmgr","checksumSHA1":"Ij8yB33E0Kk+GfTkNRoF1mG26dc=","revision":"d6e3aea90ab9f90fe8456e13fc520f43d102da4d","revisionTime":"2019-01-28T15:50:09Z","version":"=v2","versionExact":"v2"}, {"path":"github.com/circonus-labs/circonusllhist","checksumSHA1":"VbfeVqeOM+dTNxCmpvmYS0LwQn0=","revision":"7d649b46cdc2cd2ed102d350688a75a4fd7778c6","revisionTime":"2016-11-21T13:51:53Z"}, {"path":"github.com/containerd/console","checksumSHA1":"IGtuR58l2zmYRcNf8sPDlCSgovE=","origin":"github.com/opencontainers/runc/vendor/github.com/containerd/console","revision":"459bfaec1fc6c17d8bfb12d0a0f69e7e7271ed2a","revisionTime":"2018-08-23T14:46:37Z"}, {"path":"github.com/containerd/continuity/pathdriver","checksumSHA1":"GqIrOttKaO7k6HIaHQLPr3cY7rY=","origin":"github.com/docker/docker/vendor/github.com/containerd/continuity/pathdriver","revision":"320063a2ad06a1d8ada61c94c29dbe44e2d87473","revisionTime":"2018-08-16T08:14:46Z"}, From bcbc81f7ec964f91f4326521906262750fc5afc0 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 17:11:41 +0000 Subject: [PATCH 12/13] agent config: cleaner VAULT_ env lookup --- command/agent/command.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index d30958a5b5b..34567c0e280 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -256,16 +256,12 @@ func (c *Command) readConfig() *Config { // Check to see if we should read the Vault token from the environment if config.Vault.Token == "" { - if token, ok := os.LookupEnv("VAULT_TOKEN"); ok { - config.Vault.Token = token - } + config.Vault.Token = os.Getenv("VAULT_TOKEN") } // Check to see if we should read the Vault namespace from the environment if config.Vault.Namespace == "" { - if ns, ok := os.LookupEnv("VAULT_NAMESPACE"); ok { - config.Vault.Namespace = ns - } + config.Vault.Namespace = os.Getenv("VAULT_NAMESPACE") } // Default the plugin directory to be under that of the data directory if it From f8698a87a01ce86ef596ce3c83a55a015255ad30 Mon Sep 17 00:00:00 2001 From: Chris Baker Date: Fri, 5 Apr 2019 18:44:19 +0000 Subject: [PATCH 13/13] server vault client: use two vault clients, one with namespace, one without for /sys calls --- nomad/vault.go | 57 +++++++++++++++++++++++---------------------- nomad/vault_test.go | 31 ++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/nomad/vault.go b/nomad/vault.go index ed71c5e4e9d..1e252ff6948 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -15,11 +15,9 @@ import ( "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" - vapi "github.com/hashicorp/vault/api" - vaultconsts "github.com/hashicorp/vault/helper/consts" - "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" + vapi "github.com/hashicorp/vault/api" "github.com/mitchellh/mapstructure" "golang.org/x/sync/errgroup" @@ -174,9 +172,18 @@ type vaultClient struct { // limiter is used to rate limit requests to Vault limiter *rate.Limiter - // client is the Vault API client + // client is the Vault API client used for Namespace-relative integrations + // with the Vault API (anything except `/v1/sys`). If this server is not + // configured to reference a Vault namespace, this will point to the same + // client as clientSys client *vapi.Client + // clientSys is the Vault API client used for non-Namespace-relative integrations + // with the Vault API (anything involving `/v1/sys`). This client is never configured + // with a Vault namespace, because these endpoints may return errors if a namespace + // header is provided + clientSys *vapi.Client + // auth is the Vault token auth API client auth *vapi.TokenAuth @@ -253,11 +260,6 @@ func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVault return nil, err } - if c.Namespace != "" { - logger.Debug("configuring Vault namespace", "namespace", c.Namespace) - v.client.SetNamespace(c.Namespace) - } - // Launch the required goroutines v.tomb.Go(wrapNilError(v.establishConnection)) v.tomb.Go(wrapNilError(v.revokeDaemon)) @@ -311,6 +313,7 @@ func (v *vaultClient) flush() { defer v.l.Unlock() v.client = nil + v.clientSys = nil v.auth = nil v.connEstablished = false v.connEstablishedErr = nil @@ -406,28 +409,26 @@ func (v *vaultClient) buildClient() error { return err } - // Set the token and store the client + // Store the client, create/assign the /sys client + v.client = client + if v.config.Namespace != "" { + v.logger.Debug("configuring Vault namespace", "namespace", v.config.Namespace) + v.clientSys, err = vapi.NewClient(apiConf) + if err != nil { + v.logger.Error("failed to create Vault sys client and not retrying", "error", err) + return err + } + client.SetNamespace(v.config.Namespace) + } else { + v.clientSys = client + } + + // Set the token v.token = v.config.Token client.SetToken(v.token) - v.client = client v.auth = client.Auth().Token() - return nil -} -// getVaultInitStatus is used to get the init status. It first clears the namespace header, to work around an -// issue in Vault, then restores it. -func (v *vaultClient) getVaultInitStatus() (bool, error) { - v.l.Lock() - defer v.l.Unlock() - - // workaround for Vault behavior where namespace header causes /v1/sys/init (and other) endpoints to fail - if ns := v.client.Headers().Get(vaultconsts.NamespaceHeaderName); ns != "" { - v.client.SetNamespace("") - defer func() { - v.client.SetNamespace(ns) - }() - } - return v.client.Sys().InitStatus() + return nil } // establishConnection is used to make first contact with Vault. This should be @@ -447,7 +448,7 @@ OUTER: case <-retryTimer.C: // Ensure the API is reachable if !initStatus { - if _, err := v.getVaultInitStatus(); err != nil { + if _, err := v.clientSys.Sys().InitStatus(); err != nil { v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err) retryTimer.Reset(v.config.ConnectionRetryIntv) continue OUTER diff --git a/nomad/vault_test.go b/nomad/vault_test.go index 6edaed1fc72..e3f9b8692ef 100644 --- a/nomad/vault_test.go +++ b/nomad/vault_test.go @@ -176,9 +176,9 @@ func TestVaultClient_BadConfig(t *testing.T) { } } -// TestVaultClient_NamespaceSupport tests that the Vault namespace config, if present, will result in the +// TestVaultClient_WithNamespaceSupport tests that the Vault namespace config, if present, will result in the // namespace header being set on the created Vault client. -func TestVaultClient_NamespaceSupport(t *testing.T) { +func TestVaultClient_WithNamespaceSupport(t *testing.T) { t.Parallel() require := require.New(t) tr := true @@ -198,6 +198,33 @@ func TestVaultClient_NamespaceSupport(t *testing.T) { } require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) + require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) + require.NotEqual(c.clientSys, c.client) +} + +// TestVaultClient_WithoutNamespaceSupport tests that the Vault namespace config, if present, will result in the +// namespace header being set on the created Vault client. +func TestVaultClient_WithoutNamespaceSupport(t *testing.T) { + t.Parallel() + require := require.New(t) + tr := true + conf := &config.VaultConfig{ + Addr: "https://vault.service.consul:8200", + Enabled: &tr, + Token: "testvaulttoken", + Namespace: "", + } + logger := testlog.HCLogger(t) + + // Should be no error since Vault is not enabled + c, err := NewVaultClient(conf, logger, nil) + if err != nil { + t.Fatalf("failed to build vault client: %v", err) + } + + require.Equal("", c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) + require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) + require.Equal(c.clientSys, c.client) } // started separately.