Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(config): Env var config can now add new items #2653

Merged
merged 1 commit into from
Nov 19, 2021
Merged

feat(config): Env var config can now add new items #2653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 51 additions & 48 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,57 @@ module github.com/influxdata/kapacitor

go 1.17

require (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happened here? Non-blocking, just weird.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

go mod tidy the new version sorts differently, and maintains sorting.

github.com/BurntSushi/toml v0.3.1
github.com/aws/aws-sdk-go v1.38.3
github.com/benbjohnson/clock v1.1.0
github.com/benbjohnson/tmpl v1.0.0
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cespare/xxhash v1.1.0
github.com/davecgh/go-spew v1.1.1
github.com/docker/docker v20.10.5+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/eclipse/paho.mqtt.golang v1.2.0
github.com/evanphx/json-patch v4.9.0+incompatible
github.com/ghodss/yaml v1.0.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/btree v1.0.0
github.com/google/go-cmp v0.5.5
github.com/google/uuid v1.1.2
github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe
github.com/influxdata/flux v0.127.3
github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69
github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7
github.com/influxdata/influxdb v1.8.4
github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07
github.com/influxdata/pkg-config v0.2.8
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
github.com/k-sone/snmpgo v3.2.0+incompatible
github.com/mailru/easyjson v0.7.7
github.com/mitchellh/copystructure v1.0.0
github.com/mitchellh/mapstructure v1.4.1
github.com/mitchellh/reflectwalk v1.0.1
github.com/opentracing/opentracing-go v1.2.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0
github.com/prometheus/common v0.20.0
github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1
github.com/segmentio/kafka-go v0.3.10
github.com/serenize/snaker v0.0.0-20161123064335-543781d2b79b
github.com/shurcooL/markdownfmt v0.0.0-20170214213350-10aae0a270ab
github.com/stretchr/testify v1.7.0
github.com/uber/jaeger-client-go v2.28.0+incompatible
github.com/urfave/cli/v2 v2.3.0
go.etcd.io/bbolt v1.3.5
go.uber.org/zap v1.14.1
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/tools v0.1.0
google.golang.org/protobuf v1.27.1
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
)

require (
cloud.google.com/go v0.79.0 // indirect
cloud.google.com/go/bigquery v1.8.0 // indirect
Expand All @@ -20,7 +71,6 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/BurntSushi/toml v0.3.1
github.com/DATA-DOG/go-sqlmock v1.4.1 // indirect
github.com/Masterminds/semver v1.4.2 // indirect
github.com/Masterminds/sprig v2.16.0+incompatible // indirect
Expand All @@ -31,7 +81,6 @@ require (
github.com/aokoli/goutils v1.0.1 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 // indirect
github.com/armon/go-metrics v0.3.6 // indirect
github.com/aws/aws-sdk-go v1.38.3
github.com/aws/aws-sdk-go-v2 v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.1.5 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.2 // indirect
Expand All @@ -40,33 +89,23 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0 // indirect
github.com/aws/smithy-go v1.3.1 // indirect
github.com/benbjohnson/clock v1.1.0
github.com/benbjohnson/immutable v0.2.1 // indirect
github.com/benbjohnson/tmpl v1.0.0
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect
github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1
github.com/deepmap/oapi-codegen v1.6.0 // indirect
github.com/denisenkom/go-mssqldb v0.10.0 // indirect
github.com/dgryski/go-bits v0.0.0-20180113010104-bd8a69a71dc2 // indirect
github.com/dimchansky/utfbom v1.1.0 // indirect
github.com/docker/docker v20.10.5+incompatible
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0
github.com/eclipse/paho.mqtt.golang v1.2.0
github.com/editorconfig-checker/editorconfig-checker v0.0.0-20190819115812-1474bdeaf2a2 // indirect
github.com/editorconfig/editorconfig-core-go/v2 v2.1.1 // indirect
github.com/evanphx/json-patch v4.9.0+incompatible
github.com/fatih/color v1.9.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
github.com/geoffgarside/ber v0.0.0-20170306085127-854377f11dfb // indirect
github.com/ghodss/yaml v1.0.0
github.com/go-chi/chi v4.1.0+incompatible // indirect
github.com/go-kit/kit v0.10.0 // indirect
github.com/go-logfmt/logfmt v0.5.0 // indirect
Expand All @@ -75,20 +114,15 @@ require (
github.com/go-zookeeper/zk v1.0.2 // indirect
github.com/gofrs/uuid v3.3.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.0 // indirect
github.com/google/btree v1.0.0
github.com/google/flatbuffers v2.0.0+incompatible // indirect
github.com/google/go-cmp v0.5.5
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.1.2
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/googleapis/gnostic v0.4.1 // indirect
github.com/gophercloud/gophercloud v0.17.0 // indirect
github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
github.com/hashicorp/consul/api v1.8.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/hashicorp/go-hclog v0.14.1 // indirect
Expand All @@ -100,29 +134,18 @@ require (
github.com/hashicorp/serf v0.9.5 // indirect
github.com/huandu/xstrings v1.0.0 // indirect
github.com/imdario/mergo v0.3.9 // indirect
github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe
github.com/influxdata/flux v0.127.3
github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69
github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7
github.com/influxdata/influxdb v1.8.4
github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040 // indirect
github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
github.com/influxdata/pkg-config v0.2.8
github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.10 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/k-sone/snmpgo v3.2.0+incompatible
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/go-bindata v3.11.0+incompatible // indirect
github.com/kimor79/gollectd v1.0.0 // indirect
github.com/lib/pq v1.2.0 // indirect
github.com/mailru/easyjson v0.7.7
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
Expand All @@ -131,11 +154,8 @@ require (
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5 // indirect
github.com/mitchellh/copystructure v1.0.0
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.0 // indirect
github.com/mitchellh/mapstructure v1.4.1
github.com/mitchellh/reflectwalk v1.0.1
github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
Expand All @@ -144,40 +164,26 @@ require (
github.com/onsi/gomega v1.10.3 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.10.0
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.20.0
github.com/prometheus/procfs v0.6.0 // indirect
github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1
github.com/rakyll/statik v0.1.7 // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/segmentio/kafka-go v0.3.10
github.com/serenize/snaker v0.0.0-20161123064335-543781d2b79b
github.com/sergi/go-diff v1.0.0 // indirect
github.com/shurcooL/go v0.0.0-20170331015642-20b4b0a35211 // indirect
github.com/shurcooL/markdownfmt v0.0.0-20170214213350-10aae0a270ab
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/snowflakedb/gosnowflake v1.6.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0
github.com/uber-go/tally v3.3.15+incompatible // indirect
github.com/uber/athenadriver v1.1.4 // indirect
github.com/uber/jaeger-client-go v2.28.0+incompatible
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/urfave/cli/v2 v2.3.0
go.etcd.io/bbolt v1.3.5
go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.5.0 // indirect
go.uber.org/zap v1.14.1
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/mod v0.4.1 // indirect
golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826 // indirect
Expand All @@ -186,18 +192,15 @@ require (
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
golang.org/x/text v0.3.5 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.1.0
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.42.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f // indirect
google.golang.org/grpc v1.36.0 // indirect
google.golang.org/protobuf v1.27.1
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
26 changes: 22 additions & 4 deletions server/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,9 +460,7 @@ func (c *Config) applyEnvOverridesToMap(prefix string, fieldDesc string, mapValu
if spec.Kind() == reflect.Ptr {
s = spec.Elem()
}

var value string

if s.Kind() != reflect.Struct {
value = os.Getenv(prefix)
// Skip any fields we don't have a value to set
Expand Down Expand Up @@ -640,8 +638,28 @@ func (c *Config) applyEnvOverridesToStruct(prefix string, s reflect.Value) error
}
}
} else if f.Kind() == reflect.Map {
for _, k := range f.MapKeys() {
if err := c.applyEnvOverridesToMap(fmt.Sprintf("%s_%v", key, k), fieldName, f, k, f.MapIndex(k)); err != nil {
keys := map[string]reflect.Value{}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this only for kapacitor.conf ?

Copy link
Contributor Author

@docmerlin docmerlin Nov 19, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, and environmental variable config of kap.

for _, kVal := range f.MapKeys() {
keys[fmt.Sprintf("%s_%v", key, kVal)] = kVal
}

for _, env := range os.Environ() {
if !strings.HasPrefix(env, key) {
continue
}
fullKey := parseEnvKey(env)
// we need to replace k with the correctly typed k, if it is in keys.
k, ok := keys[fullKey]
if !ok {
if f.Type().Key().Kind() != reflect.String {
return errors.New("This config field does not support creation from environmental variables; key must exist in the config first")
}
k = reflect.ValueOf(strings.TrimLeft(strings.TrimPrefix(fullKey, key), "_"))
if k.String() == "" {
continue
}
}
if err := c.applyEnvOverridesToMap(fullKey, fieldName, f, k, reflect.New(f.Type().Elem())); err != nil {
return err
}
}
Expand Down
8 changes: 7 additions & 1 deletion server/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ headers = { Authorization = "your-key" }
if err := os.Setenv("KAPACITOR_HTTPPOST_0_HEADERS_Authorization", "my-key"); err != nil {
t.Fatalf("failed to set env var: %v", err)
}
if err := os.Setenv("KAPACITOR_HTTPPOST_0_HEADERS_TestHeader", "test-header-key"); err != nil {
t.Fatalf("failed to set env var: %v", err)
}

if err := c.ApplyEnvOverrides(); err != nil {
t.Fatalf("failed to apply env overrides: %v", err)
Expand All @@ -78,8 +81,11 @@ headers = { Authorization = "your-key" }
} else if c.InfluxDB[0].URLs[0] != "http://localhost:18086" {
t.Fatalf("unexpected url 0: %s", c.InfluxDB[0].URLs[0])
} else if c.HTTPPost[0].Headers["Authorization"] != "my-key" {
t.Fatalf("unexpected header Authorization: %s", c.InfluxDB[0].URLs[0])
t.Fatalf("unexpected header Authorization: expected: \"my-key\", got %s", c.HTTPPost[0].Headers["Authorization"])
} else if c.HTTPPost[0].Headers["TestHeader"] != "test-header-key" {
t.Fatalf("Could not insert TestHeader into map: %s", c.InfluxDB[0].URLs[0])
}

}

// Ensure the configuration can be parsed.
Expand Down
12 changes: 12 additions & 0 deletions server/parseenv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package server

// parseEnvKey gives us the key from the environment string.
func parseEnvKey(s string) string {
// copied from how the stdlib splits the env strings
for i := 0; i < len(s); i++ {
if s[i] == '=' {
return s[:i]
}
}
return s
}