feat(config): Env var config can now add new items (#2653)

This allows for example: adding new headers to httpposts via environmental variables
influxdata · Nov 19, 2021 · a353257 · a353257
1 parent 8c9ed5d
commit a353257
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 53 deletions.
diff --git a/go.mod b/go.mod
@@ -2,6 +2,57 @@ module github.com/influxdata/kapacitor
 
 go 1.17
 
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/aws/aws-sdk-go v1.38.3
+	github.com/benbjohnson/clock v1.1.0
+	github.com/benbjohnson/tmpl v1.0.0
+	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/cespare/xxhash v1.1.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/docker/docker v20.10.5+incompatible
+	github.com/dustin/go-humanize v1.0.0
+	github.com/eclipse/paho.mqtt.golang v1.2.0
+	github.com/evanphx/json-patch v4.9.0+incompatible
+	github.com/ghodss/yaml v1.0.0
+	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/google/btree v1.0.0
+	github.com/google/go-cmp v0.5.5
+	github.com/google/uuid v1.1.2
+	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
+	github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe
+	github.com/influxdata/flux v0.127.3
+	github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69
+	github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7
+	github.com/influxdata/influxdb v1.8.4
+	github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07
+	github.com/influxdata/pkg-config v0.2.8
+	github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
+	github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
+	github.com/k-sone/snmpgo v3.2.0+incompatible
+	github.com/mailru/easyjson v0.7.7
+	github.com/mitchellh/copystructure v1.0.0
+	github.com/mitchellh/mapstructure v1.4.1
+	github.com/mitchellh/reflectwalk v1.0.1
+	github.com/opentracing/opentracing-go v1.2.0
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/common v0.20.0
+	github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1
+	github.com/segmentio/kafka-go v0.3.10
+	github.com/serenize/snaker v0.0.0-20161123064335-543781d2b79b
+	github.com/shurcooL/markdownfmt v0.0.0-20170214213350-10aae0a270ab
+	github.com/stretchr/testify v1.7.0
+	github.com/uber/jaeger-client-go v2.28.0+incompatible
+	github.com/urfave/cli/v2 v2.3.0
+	go.etcd.io/bbolt v1.3.5
+	go.uber.org/zap v1.14.1
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
+	golang.org/x/tools v0.1.0
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
+)
+
 require (
 	cloud.google.com/go v0.79.0 // indirect
 	cloud.google.com/go/bigquery v1.8.0 // indirect
@@ -20,7 +71,6 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/BurntSushi/toml v0.3.1
 	github.com/DATA-DOG/go-sqlmock v1.4.1 // indirect
 	github.com/Masterminds/semver v1.4.2 // indirect
 	github.com/Masterminds/sprig v2.16.0+incompatible // indirect
@@ -31,7 +81,6 @@ require (
 	github.com/aokoli/goutils v1.0.1 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 // indirect
 	github.com/armon/go-metrics v0.3.6 // indirect
-	github.com/aws/aws-sdk-go v1.38.3
 	github.com/aws/aws-sdk-go-v2 v1.3.2 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.1.5 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.2 // indirect
@@ -40,33 +89,23 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0 // indirect
 	github.com/aws/smithy-go v1.3.1 // indirect
-	github.com/benbjohnson/clock v1.1.0
 	github.com/benbjohnson/immutable v0.2.1 // indirect
-	github.com/benbjohnson/tmpl v1.0.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect
 	github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible
-	github.com/cespare/xxhash v1.1.0
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
-	github.com/davecgh/go-spew v1.1.1
 	github.com/deepmap/oapi-codegen v1.6.0 // indirect
 	github.com/denisenkom/go-mssqldb v0.10.0 // indirect
 	github.com/dgryski/go-bits v0.0.0-20180113010104-bd8a69a71dc2 // indirect
 	github.com/dimchansky/utfbom v1.1.0 // indirect
-	github.com/docker/docker v20.10.5+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/dustin/go-humanize v1.0.0
-	github.com/eclipse/paho.mqtt.golang v1.2.0
 	github.com/editorconfig-checker/editorconfig-checker v0.0.0-20190819115812-1474bdeaf2a2 // indirect
 	github.com/editorconfig/editorconfig-core-go/v2 v2.1.1 // indirect
-	github.com/evanphx/json-patch v4.9.0+incompatible
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/geoffgarside/ber v0.0.0-20170306085127-854377f11dfb // indirect
-	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi v4.1.0+incompatible // indirect
 	github.com/go-kit/kit v0.10.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.0 // indirect
@@ -75,20 +114,15 @@ require (
 	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gofrs/uuid v3.3.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang/geo v0.0.0-20190916061304-5b978397cfec // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.0 // indirect
-	github.com/google/btree v1.0.0
 	github.com/google/flatbuffers v2.0.0+incompatible // indirect
-	github.com/google/go-cmp v0.5.5
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.1.2
 	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
 	github.com/googleapis/gnostic v0.4.1 // indirect
 	github.com/gophercloud/gophercloud v0.17.0 // indirect
-	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
 	github.com/hashicorp/consul/api v1.8.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-hclog v0.14.1 // indirect
@@ -100,29 +134,18 @@ require (
 	github.com/hashicorp/serf v0.9.5 // indirect
 	github.com/huandu/xstrings v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.9 // indirect
-	github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe
-	github.com/influxdata/flux v0.127.3
-	github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69
-	github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7
-	github.com/influxdata/influxdb v1.8.4
 	github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040 // indirect
-	github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07
 	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
-	github.com/influxdata/pkg-config v0.2.8
 	github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect
-	github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
-	github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/jstemmer/go-junit-report v0.9.1 // indirect
-	github.com/k-sone/snmpgo v3.2.0+incompatible
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/go-bindata v3.11.0+incompatible // indirect
 	github.com/kimor79/gollectd v1.0.0 // indirect
 	github.com/lib/pq v1.2.0 // indirect
-	github.com/mailru/easyjson v0.7.7
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
@@ -131,11 +154,8 @@ require (
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/miekg/dns v1.1.41 // indirect
 	github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5 // indirect
-	github.com/mitchellh/copystructure v1.0.0
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.1
-	github.com/mitchellh/reflectwalk v1.0.1
 	github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
@@ -144,40 +164,26 @@ require (
 	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect
-	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.10.0
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.20.0
 	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1
 	github.com/rakyll/statik v0.1.7 // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
-	github.com/segmentio/kafka-go v0.3.10
-	github.com/serenize/snaker v0.0.0-20161123064335-543781d2b79b
 	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/shurcooL/go v0.0.0-20170331015642-20b4b0a35211 // indirect
-	github.com/shurcooL/markdownfmt v0.0.0-20170214213350-10aae0a270ab
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.7.0 // indirect
 	github.com/snowflakedb/gosnowflake v1.6.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.7.0
 	github.com/uber-go/tally v3.3.15+incompatible // indirect
 	github.com/uber/athenadriver v1.1.4 // indirect
-	github.com/uber/jaeger-client-go v2.28.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/urfave/cli/v2 v2.3.0
-	go.etcd.io/bbolt v1.3.5
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.5.0 // indirect
-	go.uber.org/zap v1.14.1
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
 	golang.org/x/mod v0.4.1 // indirect
 	golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826 // indirect
@@ -186,18 +192,15 @@ require (
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.org/x/tools v0.1.0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.42.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f // indirect
 	google.golang.org/grpc v1.36.0 // indirect
-	google.golang.org/protobuf v1.27.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.51.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

diff --git a/server/config.go b/server/config.go
@@ -460,9 +460,7 @@ func (c *Config) applyEnvOverridesToMap(prefix string, fieldDesc string, mapValu
 	if spec.Kind() == reflect.Ptr {
 		s = spec.Elem()
 	}
-
 	var value string
-
 	if s.Kind() != reflect.Struct {
 		value = os.Getenv(prefix)
 		// Skip any fields we don't have a value to set
@@ -640,8 +638,28 @@ func (c *Config) applyEnvOverridesToStruct(prefix string, s reflect.Value) error
 					}
 				}
 			} else if f.Kind() == reflect.Map {
-				for _, k := range f.MapKeys() {
-					if err := c.applyEnvOverridesToMap(fmt.Sprintf("%s_%v", key, k), fieldName, f, k, f.MapIndex(k)); err != nil {
+				keys := map[string]reflect.Value{}
+				for _, kVal := range f.MapKeys() {
+					keys[fmt.Sprintf("%s_%v", key, kVal)] = kVal
+				}
+
+				for _, env := range os.Environ() {
+					if !strings.HasPrefix(env, key) {
+						continue
+					}
+					fullKey := parseEnvKey(env)
+					// we need to replace k with the correctly typed k, if it is in keys.
+					k, ok := keys[fullKey]
+					if !ok {
+						if f.Type().Key().Kind() != reflect.String {
+							return errors.New("This config field does not support creation from environmental variables; key must exist in the config first")
+						}
+						k = reflect.ValueOf(strings.TrimLeft(strings.TrimPrefix(fullKey, key), "_"))
+						if k.String() == "" {
+							continue
+						}
+					}
+					if err := c.applyEnvOverridesToMap(fullKey, fieldName, f, k, reflect.New(f.Type().Elem())); err != nil {
 						return err
 					}
 				}

diff --git a/server/config_test.go b/server/config_test.go
@@ -65,6 +65,9 @@ headers = { Authorization = "your-key" }
 	if err := os.Setenv("KAPACITOR_HTTPPOST_0_HEADERS_Authorization", "my-key"); err != nil {
 		t.Fatalf("failed to set env var: %v", err)
 	}
+	if err := os.Setenv("KAPACITOR_HTTPPOST_0_HEADERS_TestHeader", "test-header-key"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
 
 	if err := c.ApplyEnvOverrides(); err != nil {
 		t.Fatalf("failed to apply env overrides: %v", err)
@@ -78,8 +81,11 @@ headers = { Authorization = "your-key" }
 	} else if c.InfluxDB[0].URLs[0] != "http://localhost:18086" {
 		t.Fatalf("unexpected url 0: %s", c.InfluxDB[0].URLs[0])
 	} else if c.HTTPPost[0].Headers["Authorization"] != "my-key" {
-		t.Fatalf("unexpected header Authorization: %s", c.InfluxDB[0].URLs[0])
+		t.Fatalf("unexpected header Authorization: expected: \"my-key\", got %s", c.HTTPPost[0].Headers["Authorization"])
+	} else if c.HTTPPost[0].Headers["TestHeader"] != "test-header-key" {
+		t.Fatalf("Could not insert TestHeader into map: %s", c.InfluxDB[0].URLs[0])
 	}
+
 }
 
 // Ensure the configuration can be parsed.

diff --git a/server/parseenv.go b/server/parseenv.go
@@ -0,0 +1,12 @@
+package server
+
+// parseEnvKey gives us the key from the environment string.
+func parseEnvKey(s string) string {
+	// copied from how the stdlib splits the env strings
+	for i := 0; i < len(s); i++ {
+		if s[i] == '=' {
+			return s[:i]
+		}
+	}
+	return s
+}