From 5a3a47c7a4e4a48600082e820efd376116b6361c Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 11 Jun 2019 15:14:24 -0400 Subject: [PATCH 1/5] config_parse split out defaults from ParseConfig --- command/agent/config_parse.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index af9a1f5c2eb..201508f6c2a 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -14,6 +14,15 @@ import ( "github.com/hashicorp/nomad/nomad/structs/config" ) +// ParseConfigDefault returns the configuration base +func ParseConfigDefault() *Config { + return &Config{ + Consul: config.DefaultConsulConfig(), + Autopilot: config.DefaultAutopilotConfig(), + Vault: config.DefaultVaultConfig(), + } +} + func ParseConfigFile(path string) (*Config, error) { // slurp var buf bytes.Buffer @@ -36,10 +45,10 @@ func ParseConfigFile(path string) (*Config, error) { Client: &ClientConfig{ServerJoin: &ServerJoin{}}, ACL: &ACLConfig{}, Server: &ServerConfig{ServerJoin: &ServerJoin{}}, - Consul: config.DefaultConsulConfig(), - Autopilot: config.DefaultAutopilotConfig(), + Consul: &config.ConsulConfig{}, + Autopilot: &config.AutopilotConfig{}, Telemetry: &Telemetry{}, - Vault: config.DefaultVaultConfig(), + Vault: &config.VaultConfig{}, } err = hcl.Decode(c, buf.String()) From 006a9a1d454739eee21b7d8abb8b7aef1353b648 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 11 Jun 2019 15:15:52 -0400 Subject: [PATCH 2/5] config explicitly merge defaults once when using a config directory --- command/agent/config.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/command/agent/config.go b/command/agent/config.go index 773adbc1754..c5128a4772c 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -1461,8 +1461,14 @@ func LoadConfig(path string) (*Config, error) { return nil, err } + defaults := ParseConfigDefault() + if fi.IsDir() { - return LoadConfigDir(path) + config, err := LoadConfigDir(path) + if err != nil { + return nil, err + } + return defaults.Merge(config), nil } cleaned := filepath.Clean(path) @@ -1470,7 +1476,7 @@ func LoadConfig(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("Error loading %s: %s", cleaned, err) } - + config = defaults.Merge(config) config.Files = append(config.Files, cleaned) return config, nil } From e9fa8e56215b12f27b68e95738d7d70c943c75f6 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 11 Jun 2019 15:17:31 -0400 Subject: [PATCH 3/5] config_parse_test, handle defaults --- command/agent/config_parse_test.go | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 3c4e3a7533b..8d407938ee4 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -289,7 +289,7 @@ var pluginConfig = &Config{ Consul: nil, Vault: nil, TLSConfig: nil, - HTTPAPIResponseHeaders: nil, + HTTPAPIResponseHeaders: map[string]string{}, Sentinel: nil, Plugins: []*config.PluginConfig{ { @@ -355,7 +355,7 @@ var nonoptConfig = &Config{ Consul: nil, Vault: nil, TLSConfig: nil, - HTTPAPIResponseHeaders: nil, + HTTPAPIResponseHeaders: map[string]string{}, Sentinel: nil, } @@ -411,6 +411,9 @@ func TestConfig_Parse(t *testing.T) { t.Fatalf("file: %s\n\n%s", tc.File, err) } + // Merge defaults like LoadConfig does + actual = ParseConfigDefault().Merge(actual) + //panic(fmt.Sprintf("first: %+v \n second: %+v", actual.TLSConfig, tc.Result.TLSConfig)) require.EqualValues(tc.Result, removeHelperAttributes(actual)) }) @@ -548,26 +551,11 @@ var sample0 = &Config{ Token: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", ServerAutoJoin: helper.BoolToPtr(false), ClientAutoJoin: helper.BoolToPtr(false), - // Defaults - ServerServiceName: "nomad", - ServerHTTPCheckName: "Nomad Server HTTP Check", - ServerSerfCheckName: "Nomad Server Serf Check", - ServerRPCCheckName: "Nomad Server RPC Check", - ClientServiceName: "nomad-client", - ClientHTTPCheckName: "Nomad Client HTTP Check", - AutoAdvertise: helper.BoolToPtr(true), - ChecksUseAdvertise: helper.BoolToPtr(false), - Timeout: 5 * time.Second, - EnableSSL: helper.BoolToPtr(false), - VerifySSL: helper.BoolToPtr(true), }, Vault: &config.VaultConfig{ Enabled: helper.BoolToPtr(true), Role: "nomad-cluster", Addr: "http://host.example.com:8200", - // Defaults - AllowUnauthenticated: helper.BoolToPtr(true), - ConnectionRetryIntv: 30 * time.Second, }, TLSConfig: &config.TLSConfig{ EnableHTTP: true, @@ -579,10 +567,6 @@ var sample0 = &Config{ }, Autopilot: &config.AutopilotConfig{ CleanupDeadServers: helper.BoolToPtr(true), - // Defaults - ServerStabilizationTime: 10 * time.Second, - LastContactThreshold: 200 * time.Millisecond, - MaxTrailingLogs: 250, }, } From 4bfbeb649b827677f20e09fab4b850761ab5760c Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 11 Jun 2019 15:25:56 -0400 Subject: [PATCH 4/5] config merge maintains *HCL string fields used for duration conversion --- command/agent/config.go | 18 ++++++++++++++++++ nomad/structs/config/autopilot.go | 6 ++++++ nomad/structs/config/consul.go | 3 +++ 3 files changed, 27 insertions(+) diff --git a/command/agent/config.go b/command/agent/config.go index c5128a4772c..974f8d29b36 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -1090,9 +1090,15 @@ func (a *ACLConfig) Merge(b *ACLConfig) *ACLConfig { if b.TokenTTL != 0 { result.TokenTTL = b.TokenTTL } + if b.TokenTTLHCL != "" { + result.TokenTTLHCL = b.TokenTTLHCL + } if b.PolicyTTL != 0 { result.PolicyTTL = b.PolicyTTL } + if b.PolicyTTLHCL != "" { + result.PolicyTTLHCL = b.PolicyTTLHCL + } if b.ReplicationToken != "" { result.ReplicationToken = b.ReplicationToken } @@ -1139,9 +1145,15 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { if b.HeartbeatGrace != 0 { result.HeartbeatGrace = b.HeartbeatGrace } + if b.HeartbeatGraceHCL != "" { + result.HeartbeatGraceHCL = b.HeartbeatGraceHCL + } if b.MinHeartbeatTTL != 0 { result.MinHeartbeatTTL = b.MinHeartbeatTTL } + if b.MinHeartbeatTTLHCL != "" { + result.MinHeartbeatTTLHCL = b.MinHeartbeatTTLHCL + } if b.MaxHeartbeatsPerSecond != 0.0 { result.MaxHeartbeatsPerSecond = b.MaxHeartbeatsPerSecond } @@ -1151,6 +1163,9 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { if b.RetryInterval != 0 { result.RetryInterval = b.RetryInterval } + if b.RetryIntervalHCL != "" { + result.RetryIntervalHCL = b.RetryIntervalHCL + } if b.RejoinAfterLeave { result.RejoinAfterLeave = true } @@ -1232,6 +1247,9 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { if b.GCInterval != 0 { result.GCInterval = b.GCInterval } + if b.GCIntervalHCL != "" { + result.GCIntervalHCL = b.GCIntervalHCL + } if b.GCParallelDestroys != 0 { result.GCParallelDestroys = b.GCParallelDestroys } diff --git a/nomad/structs/config/autopilot.go b/nomad/structs/config/autopilot.go index c1bae189440..83f65b0f29e 100644 --- a/nomad/structs/config/autopilot.go +++ b/nomad/structs/config/autopilot.go @@ -61,9 +61,15 @@ func (a *AutopilotConfig) Merge(b *AutopilotConfig) *AutopilotConfig { if b.ServerStabilizationTime != 0 { result.ServerStabilizationTime = b.ServerStabilizationTime } + if b.ServerStabilizationTimeHCL != "" { + result.ServerStabilizationTimeHCL = b.ServerStabilizationTimeHCL + } if b.LastContactThreshold != 0 { result.LastContactThreshold = b.LastContactThreshold } + if b.LastContactThresholdHCL != "" { + result.LastContactThresholdHCL = b.LastContactThresholdHCL + } if b.MaxTrailingLogs != 0 { result.MaxTrailingLogs = b.MaxTrailingLogs } diff --git a/nomad/structs/config/consul.go b/nomad/structs/config/consul.go index 9f9cd8ecf21..750d1b916b0 100644 --- a/nomad/structs/config/consul.go +++ b/nomad/structs/config/consul.go @@ -145,6 +145,9 @@ func (a *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig { if b.Timeout != 0 { result.Timeout = b.Timeout } + if b.TimeoutHCL != "" { + result.TimeoutHCL = b.TimeoutHCL + } if b.Token != "" { result.Token = b.Token } From d1ee78ab304bad4de45f30598ff35b040fd5f324 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 11 Jun 2019 15:13:27 -0400 Subject: [PATCH 5/5] agent/testdata add a configuration directory for testing --- command/agent/config_parse_test.go | 34 +++++++++++++++++- command/agent/testdata/sample1/sample0.json | 38 +++++++++++++++++++++ command/agent/testdata/sample1/sample1.json | 38 +++++++++++++++++++++ command/agent/testdata/sample1/sample2.hcl | 19 +++++++++++ 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 command/agent/testdata/sample1/sample0.json create mode 100644 command/agent/testdata/sample1/sample1.json create mode 100644 command/agent/testdata/sample1/sample2.hcl diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 8d407938ee4..aea94556a6c 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -572,6 +572,38 @@ var sample0 = &Config{ func TestConfig_ParseSample0(t *testing.T) { c, err := ParseConfigFile("./testdata/sample0.json") - require.Nil(t, err) + require.NoError(t, err) require.EqualValues(t, sample0, c) } + +func TestConfig_ParseDir(t *testing.T) { + c, err := LoadConfig("./testdata/sample1") + require.NoError(t, err) + + // Defaults + sample1 := ParseConfigDefault().Merge(sample0) + + // Merge makes empty maps & slices rather than nil, so set those for the expected + // value + require.Nil(t, sample1.Client.Options) + sample1.Client.Options = map[string]string{} + require.Nil(t, sample1.Client.Meta) + sample1.Client.Meta = map[string]string{} + require.Nil(t, sample1.Client.ChrootEnv) + sample1.Client.ChrootEnv = map[string]string{} + require.Nil(t, sample1.Server.StartJoin) + sample1.Server.StartJoin = []string{} + + // This value added to the config file + sample1.Consul.EnableSSL = helper.BoolToPtr(true) + + // LoadDir listed the config files + require.Nil(t, sample1.Files) + sample1.Files = []string{ + "testdata/sample1/sample0.json", + "testdata/sample1/sample1.json", + "testdata/sample1/sample2.hcl", + } + + require.EqualValues(t, sample1, c) +} diff --git a/command/agent/testdata/sample1/sample0.json b/command/agent/testdata/sample1/sample0.json new file mode 100644 index 00000000000..07796e913d9 --- /dev/null +++ b/command/agent/testdata/sample1/sample0.json @@ -0,0 +1,38 @@ +{ + "acl": { + "enabled": true + }, + "bind_addr": "0.0.0.0", + "consul": { + "ssl": true, + "server_auto_join": false, + "client_auto_join": false, + "token": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + }, + "data_dir": "/opt/data/nomad/data", + "datacenter": "dc1", + "enable_syslog": true, + "leave_on_interrupt": true, + "leave_on_terminate": true, + "log_level": "INFO", + "region": "global", + "server": { + "bootstrap_expect": 3, + "enabled": true, + "encrypt": "sHck3WL6cxuhuY7Mso9BHA==", + "retry_join": [ + "10.0.0.101", + "10.0.0.102", + "10.0.0.103" + ] + }, + "syslog_facility": "LOCAL0", + "tls": { + "ca_file": "/opt/data/nomad/certs/nomad-ca.pem", + "cert_file": "/opt/data/nomad/certs/server.pem", + "http": true, + }, + "vault": { + "address": "http://host.example.com:8200", + } +} diff --git a/command/agent/testdata/sample1/sample1.json b/command/agent/testdata/sample1/sample1.json new file mode 100644 index 00000000000..8f83354d45f --- /dev/null +++ b/command/agent/testdata/sample1/sample1.json @@ -0,0 +1,38 @@ +{ + "acl": { + "enabled": true + }, + "advertise": { + "http": "host.example.com", + "rpc": "host.example.com", + "serf": "host.example.com" + }, + "bind_addr": "0.0.0.0", + "data_dir": "/opt/data/nomad/data", + "datacenter": "dc1", + "enable_syslog": true, + "leave_on_interrupt": true, + "leave_on_terminate": true, + "log_level": "INFO", + "region": "global", + "server": { + "bootstrap_expect": 3, + "enabled": true, + }, + "syslog_facility": "LOCAL0", + "telemetry": { + "collection_interval": "60s", + "disable_hostname": true, + "prometheus_metrics": true, + "publish_allocation_metrics": true, + "publish_node_metrics": true + }, + "tls": { + "key_file": "/opt/data/nomad/certs/server-key.pem", + "rpc": true, + "verify_server_hostname": true + }, + "vault": { + "create_from_role": "nomad-cluster", + } +} diff --git a/command/agent/testdata/sample1/sample2.hcl b/command/agent/testdata/sample1/sample2.hcl new file mode 100644 index 00000000000..6ad6055850c --- /dev/null +++ b/command/agent/testdata/sample1/sample2.hcl @@ -0,0 +1,19 @@ +"advertise" = { + "http" = "host.example.com" + "rpc" = "host.example.com" + "serf" = "host.example.com" +} + +"autopilot" = { + "cleanup_dead_servers" = true +} + +"consul" = { + "client_auto_join" = false + "server_auto_join" = false + "token" = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" +} + +vault = { + enabled = true +}