From 1d1e3f055f10f63456498cf82d8cb5287ad5ca41 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Tue, 6 Apr 2021 16:10:20 -0700 Subject: [PATCH 01/10] Add a Factory for ConfigSource objects --- config/internal/configsource/factory.go | 50 ++++++++++++++++++++++++ config/internal/configsource/settings.go | 50 ++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 config/internal/configsource/factory.go create mode 100644 config/internal/configsource/settings.go diff --git a/config/internal/configsource/factory.go b/config/internal/configsource/factory.go new file mode 100644 index 00000000000..29fe7b3e496 --- /dev/null +++ b/config/internal/configsource/factory.go @@ -0,0 +1,50 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "context" + + "go.uber.org/zap" + + "go.opentelemetry.io/collector/component" +) + +// CreateParams is passed to Factory.Create* functions. +type CreateParams struct { + // Logger that the factory can use during creation and can pass to the created + // ConfigSource to be used later as well. + Logger *zap.Logger + + // ApplicationStartInfo can be used to retrieve data according to version, etc. + ApplicationStartInfo component.ApplicationStartInfo +} + +// Factory is a factory interface for configuration sources. +type Factory interface { + component.Factory + + // CreateDefaultConfig creates the default configuration settings for the ConfigSource. + // This method can be called multiple times depending on the pipeline + // configuration and should not cause side-effects that prevent the creation + // of multiple instances of the ConfigSource. + // The object returned by this method needs to pass the checks implemented by + // 'configcheck.ValidateConfig'. It is recommended to have such check in the + // tests of any implementation of the Factory interface. + CreateDefaultConfig() Settings + + // CreateConfigSource creates a configuration source based on the given config. + CreateConfigSource(ctx context.Context, params CreateParams, cfg Settings) (ConfigSource, error) +} diff --git a/config/internal/configsource/settings.go b/config/internal/configsource/settings.go new file mode 100644 index 00000000000..e87835c4722 --- /dev/null +++ b/config/internal/configsource/settings.go @@ -0,0 +1,50 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "go.opentelemetry.io/collector/config" +) + +// Settings defines common settings of a ConfigSource configuration. +// Specific config sources can embed this struct and extend it with more fields if needed. +// When embedded it must be with `mapstructure:"-"` tag. +type Settings struct { + TypeVal config.Type `mapstructure:"-"` + NameVal string `mapstructure:"-"` +} + +// NewSettings return a new ConfigSourceSettings with the given type. +func NewSettings(typeVal config.Type) *Settings { + return &Settings{TypeVal: typeVal, NameVal: string(typeVal)} +} + +// Ensure that Settings satisfy the config.NamedEntity interface. +var _ config.NamedEntity = (*Settings)(nil) + +// Name gets the config source name. +func (css *Settings) Name() string { + return css.NameVal +} + +// SetName sets the config source name. +func (css *Settings) SetName(name string) { + css.NameVal = name +} + +// Type sets the config source type. +func (css *Settings) Type() config.Type { + return css.TypeVal +} From 98bbf53e27ca0232dd395cc9eb0c2ff6635f0b9b Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 8 Apr 2021 07:26:35 -0700 Subject: [PATCH 02/10] Vault related changes --- .../configsource/vaultconfigsource/config.go | 30 ++ .../vaultconfigsource/configsource.go | 56 +++ .../vaultconfigsource/configsource_test.go | 55 +++ .../configsource/vaultconfigsource/session.go | 339 +++++++++++++ .../vaultconfigsource/session_test.go | 460 ++++++++++++++++++ go.mod | 2 +- go.sum | 39 +- 7 files changed, 978 insertions(+), 3 deletions(-) create mode 100644 config/internal/configsource/vaultconfigsource/config.go create mode 100644 config/internal/configsource/vaultconfigsource/configsource.go create mode 100644 config/internal/configsource/vaultconfigsource/configsource_test.go create mode 100644 config/internal/configsource/vaultconfigsource/session.go create mode 100644 config/internal/configsource/vaultconfigsource/session_test.go diff --git a/config/internal/configsource/vaultconfigsource/config.go b/config/internal/configsource/vaultconfigsource/config.go new file mode 100644 index 00000000000..ecd82932c9a --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/config.go @@ -0,0 +1,30 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import "go.opentelemetry.io/collector/config/internal/configsource" + +type Config struct { + *configsource.Settings + // Endpoint is the address of the Vault server, typically it is set via the + // VAULT_ADDR environment variable for the Vault CLI. + Endpoint string `mapstructure:"endpoint"` + // Path is the Vault path where the secret to be retrieved is located. + Path string `mapstructure:"path"` + // Token is the token to be used to access the Vault server, typically is set + // via the VAULT_TOKEN environment variable for the Vault CLI. + Token string `mapstructure:"token"` +} diff --git a/config/internal/configsource/vaultconfigsource/configsource.go b/config/internal/configsource/vaultconfigsource/configsource.go new file mode 100644 index 00000000000..f2ee6769b38 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/configsource.go @@ -0,0 +1,56 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "context" + "time" + + "github.com/hashicorp/vault/api" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/config/internal/configsource" +) + +type vaultConfigSource struct { + client *api.Client + path string +} + +var _ configsource.ConfigSource = (*vaultConfigSource)(nil) + +func (v *vaultConfigSource) NewSession(context.Context) (configsource.Session, error) { + // TODO: Logger and poll interval should not be hard coded here but come from factory creating the config source. + return newSession(v.client, v.path, zap.NewNop(), 2*time.Second) +} + +func newConfigSource(address, token, path string) (*vaultConfigSource, error) { + // Client doesn't connect on creation and can't be closed. Keeping the same instance + // for all sessions is ok. + client, err := api.NewClient(&api.Config{ + Address: address, + }) + if err != nil { + return nil, err + } + + client.SetToken(token) + + return &vaultConfigSource{ + client: client, + path: path, + }, nil +} diff --git a/config/internal/configsource/vaultconfigsource/configsource_test.go b/config/internal/configsource/vaultconfigsource/configsource_test.go new file mode 100644 index 00000000000..6ce705ee78e --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/configsource_test.go @@ -0,0 +1,55 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVaultNewConfigSource(t *testing.T) { + tests := []struct { + name string + address string + path string + token string + wantErr bool + }{ + { + name: "minimal", + address: "https://some.server:1234/", + }, + { + name: "invalid_address", + address: "invalid\baddress", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfgSrc, err := newConfigSource(tt.address, tt.token, tt.path) + if tt.wantErr { + require.Error(t, err) + require.Nil(t, cfgSrc) + return + } + + require.NoError(t, err) + require.NotNil(t, cfgSrc) + }) + } +} diff --git a/config/internal/configsource/vaultconfigsource/session.go b/config/internal/configsource/vaultconfigsource/session.go new file mode 100644 index 00000000000..a4756eaee79 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/session.go @@ -0,0 +1,339 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/hashicorp/vault/api" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/config/internal/configsource" +) + +var errInvalidPollInterval = errors.New("poll interval must be greater than zero") + +// Error wrapper types to help with testability +type ( + errClientRead struct{ error } + errNilSecret struct{ error } + errNilSecretData struct{ error } + errBadSelector struct{ error } +) + +// vaultSession implements the configsource.Session interface. +type vaultSession struct { + logger *zap.Logger + client *api.Client + secret *api.Secret + + doneCh chan struct{} + + path string + + pollInterval time.Duration +} + +var _ configsource.Session = (*vaultSession)(nil) + +func (v *vaultSession) Retrieve(_ context.Context, selector string, _ interface{}) (configsource.Retrieved, error) { + // By default assume that watcher is not supported. The exception will be the first + // value read from the vault secret. + watchForUpdateFn := watcherNotSupported + + if v.secret == nil { + if err := v.readSecret(); err != nil { + return nil, err + } + + // The keys come all from the same secret so creating a watcher only for the + // first it is fine. + var err error + watchForUpdateFn, err = v.buildWatcherFn() + if err != nil { + return nil, err + } + } + + value := traverseToKey(v.secret.Data, selector) + if value == nil { + return nil, &errBadSelector{fmt.Errorf("no value at path %q for key %q", v.path, selector)} + } + + return newRetrieved(value, watchForUpdateFn), nil +} + +func (v *vaultSession) RetrieveEnd(context.Context) error { + return nil +} + +func (v *vaultSession) Close(context.Context) error { + close(v.doneCh) + + // Vault doesn't have a close for its client, close is completed. + return nil +} + +func newSession(client *api.Client, path string, logger *zap.Logger, pollInterval time.Duration) (*vaultSession, error) { + if pollInterval <= 0 { + return nil, errInvalidPollInterval + } + + return &vaultSession{ + logger: logger, + client: client, + path: path, + pollInterval: pollInterval, + doneCh: make(chan struct{}), + }, nil +} + +func (v *vaultSession) readSecret() error { + secret, err := v.client.Logical().Read(v.path) + if err != nil { + return &errClientRead{err} + } + + // Invalid path does not return error but a nil secret. + if secret == nil { + return &errNilSecret{fmt.Errorf("no secret found at %q", v.path)} + } + + // Incorrect path for v2 return nil data and warnings. + if secret.Data == nil { + return &errNilSecretData{fmt.Errorf("no data at %q warnings: %v", v.path, secret.Warnings)} + } + + v.secret = secret + return nil +} + +func (v *vaultSession) buildWatcherFn() (func() error, error) { + switch { + case v.secret.Renewable: + // Dynamic secret supporting renewal. + return v.buildLifetimeWatcher() + case v.secret.LeaseDuration > 0: + // Version 1 lease: re-fetch it periodically. + return v.buildV1LeaseWatcher() + default: + // Not a dynamic secret the best that can be done is polling. + return v.buildPollingWatcher() + } +} + +func (v *vaultSession) buildLifetimeWatcher() (func() error, error) { + vaultWatcher, err := v.client.NewLifetimeWatcher(&api.RenewerInput{ + Secret: v.secret, + }) + if err != nil { + return nil, err + } + + watcherFn := func() error { + go vaultWatcher.Start() + defer vaultWatcher.Stop() + + for { + select { + case <-vaultWatcher.RenewCh(): + v.logger.Debug("vault secret renewed", zap.String("path", v.path)) + case err := <-vaultWatcher.DoneCh(): + // Renewal stopped, error or not the client needs to re-fetch the configuration. + if err == nil { + return configsource.ErrValueUpdated + } + return err + case <-v.doneCh: + return configsource.ErrSessionClosed + } + } + } + + return watcherFn, nil +} + +// buildV1LeaseWatcher builds a watcher function that takes the TTL given +// by Vault and triggers the re-fetch of the secret when half of the TTl +// has passed. In principle, this could be changed to actually check if the +// values of the secret were actually changed or not. +func (v *vaultSession) buildV1LeaseWatcher() (func() error, error) { + watcherFn := func() error { + // The lease duration is a hint of time to re-fetch the values. + // The SmartAgent waits for half ot the lease duration. + updateWait := time.Duration(v.secret.LeaseDuration/2) * time.Second + select { + case <-time.After(updateWait): + // This is triggering a re-fetch. In principle this could actually + // check for changes in the values. + return configsource.ErrValueUpdated + case <-v.doneCh: + return configsource.ErrSessionClosed + } + } + + return watcherFn, nil +} + +// buildPollingWatcher builds a watcher function that monitors for changes on +// the v.secret metadata. In principle this could be done for the actual value of +// the retrieved keys. However, checking for metadata keeps this in sync with the +// SignalFx SmartAgent behavior. +func (v *vaultSession) buildPollingWatcher() (func() error, error) { + // Use the same requirements as SignalFx Smart Agent to build a polling watcher for the secret: + // + // This secret is not renewable or on a lease. If it has a + // "metadata" field and has "/data/" in the vault path, then it is + // probably a KV v2 secret. In that case, we do a poll on the + // secret's metadata to refresh it and notice if a new version is + // added to the secret. + mdValue := v.secret.Data["metadata"] + if mdValue == nil || !strings.Contains(v.path, "/data/") { + v.logger.Warn("Missing metadata to create polling watcher for vault config source", zap.String("path", v.path)) + return watcherNotSupported, nil + } + + mdMap, ok := mdValue.(map[string]interface{}) + if !ok { + v.logger.Warn("Metadata not in the expected format to create polling watcher for vault config source", zap.String("path", v.path)) + return watcherNotSupported, nil + } + + originalVersion := v.extractVersionMetadata(mdMap, "created_time", "version") + if originalVersion == nil { + v.logger.Warn("Failed to extract version metadata to create to create polling watcher for vault config source", zap.String("path", v.path)) + return watcherNotSupported, nil + } + + watcherFn := func() error { + metadataPath := strings.Replace(v.path, "/data/", "/metadata/", 1) + ticker := time.NewTicker(v.pollInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + metadataSecret, err := v.client.Logical().Read(metadataPath) + if err != nil { + // Docs are not clear about how to differentiate between temporary and permanent errors. + // Assume that the configuration needs to be re-fetched. + return fmt.Errorf("failed to read secret metadata at %q: %w", metadataPath, err) + } + + if metadataSecret == nil || metadataSecret.Data == nil { + return fmt.Errorf("no secret metadata found at %q", metadataPath) + } + + const timestampKey = "updated_time" + const versionKey = "current_version" + latestVersion := v.extractVersionMetadata(metadataSecret.Data, timestampKey, versionKey) + if latestVersion == nil { + return fmt.Errorf("secret metadata is not in the expected format for keys %q and %q", timestampKey, versionKey) + } + + // Per SmartAgent code this is enough to trigger an update but it is also possible to check if the + // the valued of the retrieved keys was changed. The current criteria may trigger updates even for + // addition of new keys to the secret. + if originalVersion.Timestamp != latestVersion.Timestamp || originalVersion.Version != latestVersion.Version { + return configsource.ErrValueUpdated + } + case <-v.doneCh: + return configsource.ErrSessionClosed + } + } + } + + return watcherFn, nil +} + +type versionMetadata struct { + Timestamp string + Version int64 +} + +func (v *vaultSession) extractVersionMetadata(metadataMap map[string]interface{}, timestampKey, versionKey string) *versionMetadata { + timestamp, ok := metadataMap[timestampKey].(string) + if !ok { + v.logger.Warn("Missing or unexpected type for timestamp on the metadata map", zap.String("key", timestampKey)) + return nil + } + + versionNumber, ok := metadataMap[versionKey].(json.Number) + if !ok { + v.logger.Warn("Missing or unexpected type for version on the metadata map", zap.String("key", versionKey)) + return nil + } + + versionInt, err := versionNumber.Int64() + if err != nil { + v.logger.Warn("Failed to parse version number into an integer", zap.String("key", versionKey), zap.String("version_number", string(versionNumber))) + return nil + } + + return &versionMetadata{ + Timestamp: timestamp, + Version: versionInt, + } +} + +// Allows key to be dot-delimited to traverse nested maps. +func traverseToKey(data map[string]interface{}, key string) interface{} { + parts := strings.Split(key, ".") + + for i := 0; ; i++ { + partVal := data[parts[i]] + if i == len(parts)-1 { + return partVal + } + + var ok bool + data, ok = partVal.(map[string]interface{}) + if !ok { + return nil + } + } +} + +func watcherNotSupported() error { + return configsource.ErrWatcherNotSupported +} + +type retrieved struct { + value interface{} + watchForUpdateFn func() error +} + +var _ configsource.Retrieved = (*retrieved)(nil) + +func (r *retrieved) Value() interface{} { + return r.value +} + +func (r *retrieved) WatchForUpdate() error { + return r.watchForUpdateFn() +} + +func newRetrieved(value interface{}, watchForUpdateFn func() error) *retrieved { + return &retrieved{ + value, + watchForUpdateFn, + } +} diff --git a/config/internal/configsource/vaultconfigsource/session_test.go b/config/internal/configsource/vaultconfigsource/session_test.go new file mode 100644 index 00000000000..75012de05e7 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/session_test.go @@ -0,0 +1,460 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/config/internal/configsource" +) + +const ( + address = "http://localhost:8200" + token = "dev_token" + vaultContainer = "vault_tests" + mongoContainer = "mongodb" + + // For the commands below whitespace is used to break the parameters into a correct slice of arguments. + + startVault = "docker run --rm -d -p 8200:8200 -e VAULT_DEV_ROOT_TOKEN_ID=" + token + " -e VAULT_TOKEN=" + token + " -e VAULT_ADDR=http://localhost:8200 --name=" + vaultContainer + " vault" + stopVault = "docker stop " + vaultContainer + + setupKVStore = "docker exec " + vaultContainer + " vault kv put secret/kv k0=v0 k1=v1" + updateKVStore = "docker exec " + vaultContainer + " vault kv put secret/kv k0=v0 k1=v1.1" + + startMongo = "docker run --rm -d -p 27017:27017 --name=" + mongoContainer + " mongo" + stopMongo = "docker stop " + mongoContainer + setupDatabaseStore = "docker exec " + vaultContainer + " vault secrets enable database" + setupMongoVaultPlugin = "docker exec " + vaultContainer + " vault write database/config/my-mongodb-database plugin_name=mongodb-database-plugin allowed_roles=my-role connection_url=mongodb://host.docker.internal:27017/admin username=\"admin\" password=\"\"" + setupMongoSecret = "docker exec " + vaultContainer + " vault write database/roles/my-role db_name=my-mongodb-database creation_statements={\"db\":\"admin\",\"roles\":[{\"role\":\"readWrite\"},{\"role\":\"read\",\"db\":\"foo\"}]} default_ttl=2s max_ttl=6s" + + createKVVer1Store = "docker exec " + vaultContainer + " vault secrets enable -version=1 kv" + setupKVVer1Store = "docker exec " + vaultContainer + " vault kv put kv/my-secret ttl=8s my-value=s3cr3t" + setupKVVer1NoTTL = "docker exec " + vaultContainer + " vault kv put kv/my-secret ttl=0s my-value=s3cr3t" +) + +func TestVaultSessionForKV(t *testing.T) { + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, setupKVStore) + + cs, err := newConfigSource(address, token, "secret/data/kv") + require.NoError(t, err) + require.NotNil(t, cs) + + s, err := cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + retrieved, err := s.Retrieve(context.Background(), "data.k0", nil) + require.NoError(t, err) + require.Equal(t, "v0", retrieved.Value().(string)) + + retrievedMetadata, err := s.Retrieve(context.Background(), "metadata.version", nil) + require.NoError(t, err) + require.NotNil(t, retrievedMetadata.Value()) + + require.NoError(t, s.RetrieveEnd(context.Background())) + + var watcherErr error + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrieved.WatchForUpdate() + }() + + require.NoError(t, s.Close(context.Background())) + + <-doneCh + require.Equal(t, configsource.ErrSessionClosed, watcherErr) +} + +func TestVaultPollingKVUpdate(t *testing.T) { + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, setupKVStore) + + cs, err := newConfigSource(address, token, "secret/data/kv") + require.NoError(t, err) + require.NotNil(t, cs) + + s, err := cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve key "k0" + retrievedK0, err := s.Retrieve(context.Background(), "data.k0", nil) + require.NoError(t, err) + require.Equal(t, "v0", retrievedK0.Value().(string)) + + // Retrieve key "k1" + retrievedK1, err := s.Retrieve(context.Background(), "data.k1", nil) + require.NoError(t, err) + require.Equal(t, "v1", retrievedK1.Value().(string)) + + // RetrieveEnd + require.NoError(t, s.RetrieveEnd(context.Background())) + + // Only the first retrieved key provides a working watcher. + require.Equal(t, configsource.ErrWatcherNotSupported, retrievedK1.WatchForUpdate()) + + var watcherErr error + var doneCh chan struct{} + doneCh = make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedK0.WatchForUpdate() + }() + + requireCmdRun(t, updateKVStore) + + // Wait for update. + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrValueUpdated) + + // Close current session. + require.NoError(t, s.Close(context.Background())) + + // Create a new session and repeat the process. + s, err = cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve key + retrievedUpdatedK1, err := s.Retrieve(context.Background(), "data.k1", nil) + require.NoError(t, err) + require.Equal(t, "v1.1", retrievedUpdatedK1.Value().(string)) + + // Wait for close. + doneCh = make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedUpdatedK1.WatchForUpdate() + }() + + require.NoError(t, s.Close(context.Background())) + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrSessionClosed) +} + +func TestVaultRenewableSecret(t *testing.T) { + // This test is based on the commands described at https://www.vaultproject.io/docs/secrets/databases/mongodb + requireCmdRun(t, startMongo) + defer requireCmdRun(t, stopMongo) + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, setupDatabaseStore) + requireCmdRun(t, setupMongoVaultPlugin) + requireCmdRun(t, setupMongoSecret) + + cs, err := newConfigSource(address, token, "database/creds/my-role") + require.NoError(t, err) + require.NotNil(t, cs) + + s, err := cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve key username, it is generated by vault no expected value. + retrievedUser, err := s.Retrieve(context.Background(), "username", nil) + require.NoError(t, err) + + // Retrieve key password, it is generated by vault no expected value. + retrievedPwd, err := s.Retrieve(context.Background(), "password", nil) + require.NoError(t, err) + + // RetrieveEnd + require.NoError(t, s.RetrieveEnd(context.Background())) + + // Only the first retrieved key provides a working watcher. + require.Equal(t, configsource.ErrWatcherNotSupported, retrievedPwd.WatchForUpdate()) + + watcherErr := retrievedUser.WatchForUpdate() + require.ErrorIs(t, watcherErr, configsource.ErrValueUpdated) + + // Close current session. + require.NoError(t, s.Close(context.Background())) + + // Create a new session and repeat the process. + s, err = cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve key username, it is generated by vault no expected value. + retrievedUpdatedUser, err := s.Retrieve(context.Background(), "username", nil) + require.NoError(t, err) + require.NotEqual(t, retrievedUser.Value(), retrievedUpdatedUser.Value()) + + // Retrieve password and check that it changed. + retrievedUpdatedPwd, err := s.Retrieve(context.Background(), "password", nil) + require.NoError(t, err) + require.NotEqual(t, retrievedPwd.Value(), retrievedUpdatedPwd.Value()) + + // Wait for close. + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedUpdatedUser.WatchForUpdate() + }() + + runtime.Gosched() + require.NoError(t, s.Close(context.Background())) + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrSessionClosed) +} + +func TestVaultV1SecretWithTTL(t *testing.T) { + // This test is based on the commands described at https://www.vaultproject.io/docs/secrets/kv/kv-v1 + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, createKVVer1Store) + requireCmdRun(t, setupKVVer1Store) + + cs, err := newConfigSource(address, token, "kv/my-secret") + require.NoError(t, err) + require.NotNil(t, cs) + + s, err := cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve value + retrievedValue, err := s.Retrieve(context.Background(), "my-value", nil) + require.NoError(t, err) + require.Equal(t, "s3cr3t", retrievedValue.Value().(string)) + + // RetrieveEnd + require.NoError(t, s.RetrieveEnd(context.Background())) + + var watcherErr error + var doneCh chan struct{} + doneCh = make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedValue.WatchForUpdate() + }() + + // Wait for update. + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrValueUpdated) + + // Close current session. + require.NoError(t, s.Close(context.Background())) + + // Create a new session and repeat the process. + s, err = cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve value + retrievedValue, err = s.Retrieve(context.Background(), "my-value", nil) + require.NoError(t, err) + require.Equal(t, "s3cr3t", retrievedValue.Value().(string)) + + // Wait for close. + doneCh = make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedValue.WatchForUpdate() + }() + + require.NoError(t, s.Close(context.Background())) + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrSessionClosed) +} + +func TestVaultV1NonWatchableSecret(t *testing.T) { + // This test is based on the commands described at https://www.vaultproject.io/docs/secrets/kv/kv-v1 + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, createKVVer1Store) + requireCmdRun(t, setupKVVer1NoTTL) + + cs, err := newConfigSource(address, token, "kv/my-secret") + require.NoError(t, err) + require.NotNil(t, cs) + + s, err := cs.NewSession(context.Background()) + require.NoError(t, err) + require.NotNil(t, s) + + // Retrieve value + retrievedValue, err := s.Retrieve(context.Background(), "my-value", nil) + require.NoError(t, err) + require.Equal(t, "s3cr3t", retrievedValue.Value().(string)) + + // RetrieveEnd + require.NoError(t, s.RetrieveEnd(context.Background())) + + var watcherErr error + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + watcherErr = retrievedValue.WatchForUpdate() + }() + + // Wait for update. + <-doneCh + require.ErrorIs(t, watcherErr, configsource.ErrWatcherNotSupported) + + // Close current session. + require.NoError(t, s.Close(context.Background())) +} + +func TestVaultRetrieveErrors(t *testing.T) { + requireCmdRun(t, startVault) + defer requireCmdRun(t, stopVault) + requireCmdRun(t, setupKVStore) + + ctx := context.Background() + + tests := []struct { + err error + name string + path string + token string + selector string + }{ + { + name: "bad_token", + path: "secret/data/kv", + token: "bad_test_token", + err: &errClientRead{}, + }, + { + name: "non_existent_path", + path: "made_up_path/data/kv", + err: &errNilSecret{}, + }, + { + name: "v2_missing_data_on_path", + path: "secret/kv", + err: &errNilSecretData{}, + }, + { + name: "bad_selector", + path: "secret/data/kv", + selector: "data.missing", + err: &errBadSelector{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testToken := token + if tt.token != "" { + testToken = tt.token + } + + cfgSrc, err := newConfigSource(address, testToken, tt.path) + require.NoError(t, err) + require.NotNil(t, cfgSrc) + + s, err := cfgSrc.NewSession(ctx) + require.NoError(t, err) + require.NotNil(t, s) + + defer func() { + assert.NoError(t, s.Close(ctx)) + }() + defer func() { + assert.NoError(t, s.RetrieveEnd(ctx)) + }() + + r, err := s.Retrieve(ctx, tt.selector, nil) + require.Error(t, err) + require.IsType(t, tt.err, err) + require.Nil(t, r) + }) + } +} + +func requireCmdRun(t *testing.T, cli string) { + parts := strings.Split(cli, " ") + cmd := exec.Command(parts[0], parts[1:]...) // #nosec + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + time.Sleep(500 * time.Millisecond) + if err != nil { + err = fmt.Errorf("cmd.Run() %s %v failed %w. stdout: %q stderr: %q", cmd.Path, cmd.Args, err, stdout.String(), stderr.String()) + } + require.NoError(t, err) +} + +func Test_vaultSession_extractVersionMetadata(t *testing.T) { + tests := []struct { + metadataMap map[string]interface{} + expectedMd *versionMetadata + name string + }{ + { + name: "typical", + metadataMap: map[string]interface{}{ + "tsKey": "2021-04-02T22:30:51.4733477Z", + "verKey": json.Number("1"), + }, + expectedMd: &versionMetadata{ + Timestamp: "2021-04-02T22:30:51.4733477Z", + Version: 1, + }, + }, + { + name: "missing_expected_timestamp", + metadataMap: map[string]interface{}{ + "otherKey": "2021-04-02T22:30:51.4733477Z", + "verKey": json.Number("1"), + }, + }, + { + name: "missing_expected_version", + metadataMap: map[string]interface{}{ + "tsKey": "2021-04-02T22:30:51.4733477Z", + "otherKey": json.Number("1"), + }, + }, + { + name: "incorrect_version_format", + metadataMap: map[string]interface{}{ + "tsKey": "2021-04-02T22:30:51.4733477Z", + "verKey": json.Number("not_a_number"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &vaultSession{ + logger: zap.NewNop(), + } + + metadata := v.extractVersionMetadata(tt.metadataMap, "tsKey", "verKey") + assert.Equal(t, tt.expectedMd, metadata) + }) + } +} diff --git a/go.mod b/go.mod index 76e2931983f..df8ad4b3398 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/google/uuid v1.2.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 + github.com/hashicorp/vault/api v1.1.0 github.com/jaegertracing/jaeger v1.22.0 github.com/leoluk/perflib_exporter v0.1.0 github.com/openzipkin/zipkin-go v0.2.5 @@ -49,6 +50,5 @@ require ( google.golang.org/genproto v0.0.0-20210302174412-5ede27ff9881 google.golang.org/grpc v1.36.1 google.golang.org/protobuf v1.26.0 - gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index d546d18de34..67b3bfc0208 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= @@ -118,8 +119,9 @@ github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.0 h1:B7AQgHi8QSEi4uHu7Sbsga+IJDU+CENgjxoo81vDUqU= +github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= @@ -129,6 +131,7 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.37.8 h1:9kywcbuz6vQuTf+FD+U7FshafrHzmqUCjgAEiLuIJ8U= @@ -162,6 +165,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -242,6 +247,7 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= @@ -267,6 +273,7 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -274,6 +281,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-ldap/ldap/v3 v3.1.3/go.mod h1:3rbOH3jRS2u6jg2rJnKAMLE/xQyCKIveG2Sa/Cohzb8= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= @@ -372,6 +380,9 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -532,20 +543,28 @@ github.com/hashicorp/consul/sdk v0.7.0 h1:H6R9d008jDcHPQPAqPNuydAshJ4v5/8URdFnUv github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.15.0 h1:qMuK0wxsoW4D0ddCCYwPSTm4KQv1X1ke3WmPWZ0Mvsk= github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-plugin v1.4.0/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -557,6 +576,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -574,6 +594,10 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/vault/api v1.1.0 h1:QcxC7FuqEl0sZaIjcXB/kNEeBa0DH5z57qbWBvZwLC4= +github.com/hashicorp/vault/api v1.1.0/go.mod h1:R3Umvhlxi2TN7Ex2hzOowyeNb+SfbVWI973N+ctaFMk= +github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267 h1:e1ok06zGrWJW91rzRroyl5nRNqraaBe4d5hiKcVZuHM= +github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hetznercloud/hcloud-go v1.23.1 h1:SkYdCa6x458cMSDz5GI18iPz5j2hicACiDP6J/s/bTs= @@ -712,6 +736,7 @@ github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -729,6 +754,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -798,8 +824,9 @@ github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.5 h1:UwtQQx2pyPIgWYHRg+epgdx1/HnBQTgN3/oIYEJTQzU= github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -898,6 +925,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 h1:AJNDS0kP60X8wwWFvbLPwDuojxubj9pbfK7pjHw0vKg= github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -982,6 +1011,7 @@ github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZ github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= @@ -1058,6 +1088,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1196,6 +1227,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1276,6 +1308,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1438,12 +1471,14 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210302174412-5ede27ff9881 h1:SYuy3hIRsBIROE0aZwsJZOEJNC/n9/p0FmLEU9C31AE= google.golang.org/genproto v0.0.0-20210302174412-5ede27ff9881/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= From dd24098bc95fa5f4c191e54de9803203a7d80a07 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 8 Apr 2021 07:27:25 -0700 Subject: [PATCH 03/10] Factories and configuration --- config/internal/configsource/factory.go | 7 +- config/internal/configsource/parser.go | 110 +++++++++++++++++++++++ config/internal/configsource/settings.go | 8 +- 3 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 config/internal/configsource/parser.go diff --git a/config/internal/configsource/factory.go b/config/internal/configsource/factory.go index 29fe7b3e496..b5b605b23cf 100644 --- a/config/internal/configsource/factory.go +++ b/config/internal/configsource/factory.go @@ -20,6 +20,7 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" ) // CreateParams is passed to Factory.Create* functions. @@ -43,8 +44,10 @@ type Factory interface { // The object returned by this method needs to pass the checks implemented by // 'configcheck.ValidateConfig'. It is recommended to have such check in the // tests of any implementation of the Factory interface. - CreateDefaultConfig() Settings + CreateDefaultConfig() ConfigSettings // CreateConfigSource creates a configuration source based on the given config. - CreateConfigSource(ctx context.Context, params CreateParams, cfg Settings) (ConfigSource, error) + CreateConfigSource(ctx context.Context, params CreateParams, cfg ConfigSettings) (ConfigSource, error) } + +type Factories map[config.Type]Factory diff --git a/config/internal/configsource/parser.go b/config/internal/configsource/parser.go new file mode 100644 index 00000000000..e6c20b543e1 --- /dev/null +++ b/config/internal/configsource/parser.go @@ -0,0 +1,110 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "context" + "fmt" + + "github.com/spf13/cast" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configparser" +) + +const ( + configSourcesKey = "config_sources" +) + +// Private error types to help with testability. +type ( + errorInvalidTypeAndNameKey struct{ error } + errorUnknownType struct{ error } + errorUnmarshalError struct{ error } + errorDuplicateName struct{ error } +) + +// Load reads the configuration for ConfigSource objects from the given parser and returns a map +// from the full name of config sources to the respective ConfigSettings. +func Load(ctx context.Context, v *config.Parser, factories Factories) (map[string]ConfigSettings, error) { + + cfgSrcSettings, err := loadSettings(cast.ToStringMap(v.Get(configSourcesKey)), factories) + if err != nil { + return nil, err + } + + return cfgSrcSettings, nil +} + +// Build builds the ConfigSource objects according to the given ConfigSettings. +func Build(ctx context.Context, configSourcesSettings map[string]ConfigSettings, params CreateParams, factories Factories) (map[string]ConfigSource, error) { + cfgSources := make(map[string]ConfigSource, len(configSourcesSettings)) + for fullName, cfgSrcSettings := range configSourcesSettings { + // If we have the setting we also have the factory. + factory, ok := factories[cfgSrcSettings.Type()] + if !ok { + return nil, errorUnknownType{fmt.Errorf("unknown %s config source type for %s", cfgSrcSettings.Type(), fullName)} + } + + cfgSrc, err := factory.CreateConfigSource(ctx, params, cfgSrcSettings) + if err != nil { + return nil, fmt.Errorf("failed to create config source %s: %v", fullName, err) + } + cfgSources[fullName] = cfgSrc + } + + return cfgSources, nil +} + +func loadSettings(css map[string]interface{}, factories Factories) (map[string]ConfigSettings, error) { + // Prepare resulting map. + cfgSrcToSettings := make(map[string]ConfigSettings) + + // Iterate over extensions and create a config for each. + for key, value := range css { + settingsParser := config.NewParserFromStringMap(cast.ToStringMap(value)) + // TODO: expand env vars. + + // Decode the key into type and fullName components. + typeStr, fullName, err := configparser.DecodeTypeAndName(key) + if err != nil { + return nil, &errorInvalidTypeAndNameKey{fmt.Errorf("invalid %s type and name key %q: %v", configSourcesKey, key, err)} + } + + // Find the factory based on "type" that we read from config source. + factory := factories[typeStr] + if factory == nil { + return nil, &errorUnknownType{fmt.Errorf("unknown %s type %q for %s", configSourcesKey, typeStr, fullName)} + } + + // Create the default config. + cfgSrcSettings := factory.CreateDefaultConfig() + cfgSrcSettings.SetName(fullName) + + // Now that the default settings struct is created we can Unmarshal into it + // and it will apply user-defined config on top of the default. + if err := settingsParser.UnmarshalExact(&cfgSrcSettings); err != nil { + return nil, errorUnmarshalError{fmt.Errorf("error reading %s configuration for %s: %v", configSourcesKey, fullName, err)} + } + + if cfgSrcToSettings[fullName] != nil { + return nil, errorDuplicateName{fmt.Errorf("duplicate %s name %s", configSourcesKey, fullName)} + } + + cfgSrcToSettings[fullName] = cfgSrcSettings + } + + return cfgSrcToSettings, nil +} diff --git a/config/internal/configsource/settings.go b/config/internal/configsource/settings.go index e87835c4722..5057f3f76b5 100644 --- a/config/internal/configsource/settings.go +++ b/config/internal/configsource/settings.go @@ -18,6 +18,10 @@ import ( "go.opentelemetry.io/collector/config" ) +type ConfigSettings interface { + config.NamedEntity +} + // Settings defines common settings of a ConfigSource configuration. // Specific config sources can embed this struct and extend it with more fields if needed. // When embedded it must be with `mapstructure:"-"` tag. @@ -31,8 +35,8 @@ func NewSettings(typeVal config.Type) *Settings { return &Settings{TypeVal: typeVal, NameVal: string(typeVal)} } -// Ensure that Settings satisfy the config.NamedEntity interface. -var _ config.NamedEntity = (*Settings)(nil) +// Ensure that Settings satisfy the ConfigSettings interface. +var _ ConfigSettings = (*Settings)(nil) // Name gets the config source name. func (css *Settings) Name() string { From 822f2c3637a844e1b67bac5822c8a893ea449079 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 8 Apr 2021 16:56:59 -0700 Subject: [PATCH 04/10] Separate builder and parser and add parser tests --- config/internal/configsource/builder.go | 43 ++++++ config/internal/configsource/parser.go | 37 ++--- config/internal/configsource/parser_test.go | 135 ++++++++++++++++++ .../configsource/testdata/bad_name.yaml | 2 + .../configsource/testdata/basic_config.yaml | 5 + .../testdata/duplicated_name.yaml | 6 + .../configsource/testdata/unknown_field.yaml | 3 + 7 files changed, 203 insertions(+), 28 deletions(-) create mode 100644 config/internal/configsource/builder.go create mode 100644 config/internal/configsource/parser_test.go create mode 100644 config/internal/configsource/testdata/bad_name.yaml create mode 100644 config/internal/configsource/testdata/basic_config.yaml create mode 100644 config/internal/configsource/testdata/duplicated_name.yaml create mode 100644 config/internal/configsource/testdata/unknown_field.yaml diff --git a/config/internal/configsource/builder.go b/config/internal/configsource/builder.go new file mode 100644 index 00000000000..023759e2cb1 --- /dev/null +++ b/config/internal/configsource/builder.go @@ -0,0 +1,43 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "context" + "fmt" + + "go.uber.org/zap" +) + +// Build builds the ConfigSource objects according to the given ConfigSettings. +func Build(ctx context.Context, configSourcesSettings map[string]ConfigSettings, params CreateParams, factories Factories) (map[string]ConfigSource, error) { + cfgSources := make(map[string]ConfigSource, len(configSourcesSettings)) + for fullName, cfgSrcSettings := range configSourcesSettings { + // If we have the setting we also have the factory. + factory, ok := factories[cfgSrcSettings.Type()] + if !ok { + return nil, errUnknownType{fmt.Errorf("unknown %s config source type for %s", cfgSrcSettings.Type(), fullName)} + } + + params.Logger = params.Logger.With(zap.String("config_source", fullName)) + cfgSrc, err := factory.CreateConfigSource(ctx, params, cfgSrcSettings) + if err != nil { + return nil, fmt.Errorf("failed to create config source %s: %v", fullName, err) + } + cfgSources[fullName] = cfgSrc + } + + return cfgSources, nil +} diff --git a/config/internal/configsource/parser.go b/config/internal/configsource/parser.go index e6c20b543e1..202ddba7e68 100644 --- a/config/internal/configsource/parser.go +++ b/config/internal/configsource/parser.go @@ -30,10 +30,10 @@ const ( // Private error types to help with testability. type ( - errorInvalidTypeAndNameKey struct{ error } - errorUnknownType struct{ error } - errorUnmarshalError struct{ error } - errorDuplicateName struct{ error } + errInvalidTypeAndNameKey struct{ error } + errUnknownType struct{ error } + errUnmarshalError struct{ error } + errDuplicateName struct{ error } ) // Load reads the configuration for ConfigSource objects from the given parser and returns a map @@ -48,26 +48,6 @@ func Load(ctx context.Context, v *config.Parser, factories Factories) (map[strin return cfgSrcSettings, nil } -// Build builds the ConfigSource objects according to the given ConfigSettings. -func Build(ctx context.Context, configSourcesSettings map[string]ConfigSettings, params CreateParams, factories Factories) (map[string]ConfigSource, error) { - cfgSources := make(map[string]ConfigSource, len(configSourcesSettings)) - for fullName, cfgSrcSettings := range configSourcesSettings { - // If we have the setting we also have the factory. - factory, ok := factories[cfgSrcSettings.Type()] - if !ok { - return nil, errorUnknownType{fmt.Errorf("unknown %s config source type for %s", cfgSrcSettings.Type(), fullName)} - } - - cfgSrc, err := factory.CreateConfigSource(ctx, params, cfgSrcSettings) - if err != nil { - return nil, fmt.Errorf("failed to create config source %s: %v", fullName, err) - } - cfgSources[fullName] = cfgSrc - } - - return cfgSources, nil -} - func loadSettings(css map[string]interface{}, factories Factories) (map[string]ConfigSettings, error) { // Prepare resulting map. cfgSrcToSettings := make(map[string]ConfigSettings) @@ -75,18 +55,19 @@ func loadSettings(css map[string]interface{}, factories Factories) (map[string]C // Iterate over extensions and create a config for each. for key, value := range css { settingsParser := config.NewParserFromStringMap(cast.ToStringMap(value)) + // TODO: expand env vars. // Decode the key into type and fullName components. typeStr, fullName, err := configparser.DecodeTypeAndName(key) if err != nil { - return nil, &errorInvalidTypeAndNameKey{fmt.Errorf("invalid %s type and name key %q: %v", configSourcesKey, key, err)} + return nil, &errInvalidTypeAndNameKey{fmt.Errorf("invalid %s type and name key %q: %v", configSourcesKey, key, err)} } // Find the factory based on "type" that we read from config source. factory := factories[typeStr] if factory == nil { - return nil, &errorUnknownType{fmt.Errorf("unknown %s type %q for %s", configSourcesKey, typeStr, fullName)} + return nil, &errUnknownType{fmt.Errorf("unknown %s type %q for %s", configSourcesKey, typeStr, fullName)} } // Create the default config. @@ -96,11 +77,11 @@ func loadSettings(css map[string]interface{}, factories Factories) (map[string]C // Now that the default settings struct is created we can Unmarshal into it // and it will apply user-defined config on top of the default. if err := settingsParser.UnmarshalExact(&cfgSrcSettings); err != nil { - return nil, errorUnmarshalError{fmt.Errorf("error reading %s configuration for %s: %v", configSourcesKey, fullName, err)} + return nil, &errUnmarshalError{fmt.Errorf("error reading %s configuration for %s: %v", configSourcesKey, fullName, err)} } if cfgSrcToSettings[fullName] != nil { - return nil, errorDuplicateName{fmt.Errorf("duplicate %s name %s", configSourcesKey, fullName)} + return nil, &errDuplicateName{fmt.Errorf("duplicate %s name %s", configSourcesKey, fullName)} } cfgSrcToSettings[fullName] = cfgSrcSettings diff --git a/config/internal/configsource/parser_test.go b/config/internal/configsource/parser_test.go new file mode 100644 index 00000000000..b771a95b955 --- /dev/null +++ b/config/internal/configsource/parser_test.go @@ -0,0 +1,135 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "context" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/config" +) + +func TestConfigSourceParser(t *testing.T) { + ctx := context.Background() + + testFactories := Factories{ + "tstcfgsrc": &mockCfgSrcFactory{}, + } + tests := []struct { + name string + file string + factories Factories + expected map[string]ConfigSettings + wantErr error + }{ + { + name: "basic_config", + file: "basic_config", + factories: testFactories, + expected: map[string]ConfigSettings{ + "tstcfgsrc": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc", + }, + Endpoint: "some_endpoint", + Token: "some_token", + }, + "tstcfgsrc/named": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc/named", + }, + Endpoint: "default_endpoint", + }, + }, + }, + { + name: "bad_name", + file: "bad_name", + factories: testFactories, + wantErr: &errInvalidTypeAndNameKey{}, + }, + { + name: "missing_factory", + file: "basic_config", + factories: Factories{ + "not_in_basic_config": &mockCfgSrcFactory{}, + }, + wantErr: &errUnknownType{}, + }, + { + name: "unknown_field", + file: "unknown_field", + factories: testFactories, + wantErr: &errUnmarshalError{}, + }, + { + name: "duplicated_name", + file: "duplicated_name", + factories: testFactories, + wantErr: &errDuplicateName{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfgFile := path.Join("testdata", tt.file+".yaml") + v, err := config.NewParserFromFile(cfgFile) + require.NoError(t, err) + + cfgSrcSettings, err := Load(ctx, v, tt.factories) + require.IsType(t, tt.wantErr, err) + assert.Equal(t, tt.expected, cfgSrcSettings) + }) + } +} + +type mockCfgSrcSettings struct { + Settings + Endpoint string `mapstructure:"endpoint"` + Token string `mapstructure:"token"` +} + +var _ (ConfigSettings) = (*mockCfgSrcSettings)(nil) + +type mockCfgSrcFactory struct { + ErrOnCreateConfigSource error +} + +var _ (Factory) = (*mockCfgSrcFactory)(nil) + +func (m *mockCfgSrcFactory) Type() config.Type { + return "tstcfgsrc" +} + +func (m *mockCfgSrcFactory) CreateDefaultConfig() ConfigSettings { + return &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + }, + Endpoint: "default_endpoint", + } +} + +func (m *mockCfgSrcFactory) CreateConfigSource(ctx context.Context, params CreateParams, cfg ConfigSettings) (ConfigSource, error) { + if m.ErrOnCreateConfigSource != nil { + return nil, m.ErrOnCreateConfigSource + } + return &testConfigSource{}, nil +} diff --git a/config/internal/configsource/testdata/bad_name.yaml b/config/internal/configsource/testdata/bad_name.yaml new file mode 100644 index 00000000000..346e21e70f3 --- /dev/null +++ b/config/internal/configsource/testdata/bad_name.yaml @@ -0,0 +1,2 @@ +config_sources: + tstcfgsrc/: diff --git a/config/internal/configsource/testdata/basic_config.yaml b/config/internal/configsource/testdata/basic_config.yaml new file mode 100644 index 00000000000..6b21c8a4838 --- /dev/null +++ b/config/internal/configsource/testdata/basic_config.yaml @@ -0,0 +1,5 @@ +config_sources: + tstcfgsrc: + endpoint: some_endpoint + token: some_token + tstcfgsrc/named: diff --git a/config/internal/configsource/testdata/duplicated_name.yaml b/config/internal/configsource/testdata/duplicated_name.yaml new file mode 100644 index 00000000000..019c65e9841 --- /dev/null +++ b/config/internal/configsource/testdata/duplicated_name.yaml @@ -0,0 +1,6 @@ +config_sources: + tstcfgsrc/name: + endpoint: some_endpoint + token: some_token + tstcfgsrc/ name : + endpoint: some_endpoint diff --git a/config/internal/configsource/testdata/unknown_field.yaml b/config/internal/configsource/testdata/unknown_field.yaml new file mode 100644 index 00000000000..aca3ee8bda3 --- /dev/null +++ b/config/internal/configsource/testdata/unknown_field.yaml @@ -0,0 +1,3 @@ +config_sources: + tstcfgsrc: + unkwon_field: some_value From a89cc3cd96ab822a1ffd9ca3e0acbd7526275763 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 8 Apr 2021 23:25:19 -0700 Subject: [PATCH 05/10] Added builder and parser for configsource plus respective to vaultconfigsource --- config/internal/configsource/builder.go | 21 ++- config/internal/configsource/builder_test.go | 152 ++++++++++++++++++ config/internal/configsource/parser.go | 2 +- config/internal/configsource/parser_test.go | 10 +- .../configsource/vaultconfigsource/config.go | 14 +- .../vaultconfigsource/config_test.go | 73 +++++++++ .../vaultconfigsource/configsource.go | 21 +-- .../vaultconfigsource/configsource_test.go | 20 ++- .../configsource/vaultconfigsource/factory.go | 85 ++++++++++ .../vaultconfigsource/factory_test.go | 99 ++++++++++++ .../vaultconfigsource/session_test.go | 60 ++++++- .../vaultconfigsource/testdata/config.yaml | 10 ++ 12 files changed, 536 insertions(+), 31 deletions(-) create mode 100644 config/internal/configsource/builder_test.go create mode 100644 config/internal/configsource/vaultconfigsource/config_test.go create mode 100644 config/internal/configsource/vaultconfigsource/factory.go create mode 100644 config/internal/configsource/vaultconfigsource/factory_test.go create mode 100644 config/internal/configsource/vaultconfigsource/testdata/config.yaml diff --git a/config/internal/configsource/builder.go b/config/internal/configsource/builder.go index 023759e2cb1..186089322a6 100644 --- a/config/internal/configsource/builder.go +++ b/config/internal/configsource/builder.go @@ -21,6 +21,12 @@ import ( "go.uber.org/zap" ) +// Private error types to help with testability. +type ( + errConfigSourceCreation struct{ error } + errFactoryCreatedNil struct{ error } +) + // Build builds the ConfigSource objects according to the given ConfigSettings. func Build(ctx context.Context, configSourcesSettings map[string]ConfigSettings, params CreateParams, factories Factories) (map[string]ConfigSource, error) { cfgSources := make(map[string]ConfigSource, len(configSourcesSettings)) @@ -28,14 +34,25 @@ func Build(ctx context.Context, configSourcesSettings map[string]ConfigSettings, // If we have the setting we also have the factory. factory, ok := factories[cfgSrcSettings.Type()] if !ok { - return nil, errUnknownType{fmt.Errorf("unknown %s config source type for %s", cfgSrcSettings.Type(), fullName)} + return nil, &errUnknownType{ + fmt.Errorf("unknown %s config source type for %s", cfgSrcSettings.Type(), fullName), + } } params.Logger = params.Logger.With(zap.String("config_source", fullName)) cfgSrc, err := factory.CreateConfigSource(ctx, params, cfgSrcSettings) if err != nil { - return nil, fmt.Errorf("failed to create config source %s: %v", fullName, err) + return nil, &errConfigSourceCreation{ + fmt.Errorf("failed to create config source %s: %w", fullName, err), + } } + + if cfgSrc == nil { + return nil, &errFactoryCreatedNil{ + fmt.Errorf("factory for %q produced a nil extension", fullName), + } + } + cfgSources[fullName] = cfgSrc } diff --git a/config/internal/configsource/builder_test.go b/config/internal/configsource/builder_test.go new file mode 100644 index 00000000000..329fe74142d --- /dev/null +++ b/config/internal/configsource/builder_test.go @@ -0,0 +1,152 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configsource + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" +) + +func TestConfigSourceBuild(t *testing.T) { + ctx := context.Background() + params := CreateParams{ + Logger: zap.NewNop(), + ApplicationStartInfo: component.ApplicationStartInfo{}, + } + + testFactories := Factories{ + "tstcfgsrc": &mockCfgSrcFactory{}, + } + + tests := []struct { + name string + configSettings map[string]ConfigSettings + factories Factories + expectedCfgSources map[string]ConfigSource + wantErr error + }{ + { + name: "unknown_config_source", + configSettings: map[string]ConfigSettings{ + "tstcfgsrc": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "unknown_config_source", + NameVal: "tstcfgsrc", + }, + }, + }, + factories: testFactories, + wantErr: &errUnknownType{}, + }, + { + name: "creation_error", + configSettings: map[string]ConfigSettings{ + "tstcfgsrc": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc", + }, + }, + }, + factories: Factories{ + "tstcfgsrc": &mockCfgSrcFactory{ + ErrOnCreateConfigSource: errors.New("forced test error"), + }, + }, + wantErr: &errConfigSourceCreation{}, + }, + { + name: "factory_return_nil", + configSettings: map[string]ConfigSettings{ + "tstcfgsrc": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc", + }, + }, + }, + factories: Factories{ + "tstcfgsrc": &mockNilCfgSrcFactory{}, + }, + wantErr: &errFactoryCreatedNil{}, + }, + { + name: "base_case", + configSettings: map[string]ConfigSettings{ + "tstcfgsrc/named": &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc/named", + }, + Endpoint: "some_endpoint", + Token: "some_token", + }, + }, + factories: testFactories, + expectedCfgSources: map[string]ConfigSource{ + "tstcfgsrc/named": &testConfigSource{ + ValueMap: map[string]valueEntry{ + "tstcfgsrc/named": { + Value: &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + NameVal: "tstcfgsrc/named", + }, + Endpoint: "some_endpoint", + Token: "some_token", + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builtCfgSources, err := Build(ctx, tt.configSettings, params, tt.factories) + require.IsType(t, tt.wantErr, err) + require.Equal(t, tt.expectedCfgSources, builtCfgSources) + }) + } +} + +type mockNilCfgSrcFactory struct{} + +func (m *mockNilCfgSrcFactory) Type() config.Type { + return "tstcfgsrc" +} + +var _ (Factory) = (*mockNilCfgSrcFactory)(nil) + +func (m *mockNilCfgSrcFactory) CreateDefaultConfig() ConfigSettings { + return &mockCfgSrcSettings{ + Settings: Settings{ + TypeVal: "tstcfgsrc", + }, + Endpoint: "default_endpoint", + } +} + +func (m *mockNilCfgSrcFactory) CreateConfigSource(context.Context, CreateParams, ConfigSettings) (ConfigSource, error) { + return nil, nil +} diff --git a/config/internal/configsource/parser.go b/config/internal/configsource/parser.go index 202ddba7e68..ecf789a2c45 100644 --- a/config/internal/configsource/parser.go +++ b/config/internal/configsource/parser.go @@ -38,7 +38,7 @@ type ( // Load reads the configuration for ConfigSource objects from the given parser and returns a map // from the full name of config sources to the respective ConfigSettings. -func Load(ctx context.Context, v *config.Parser, factories Factories) (map[string]ConfigSettings, error) { +func Load(_ context.Context, v *config.Parser, factories Factories) (map[string]ConfigSettings, error) { cfgSrcSettings, err := loadSettings(cast.ToStringMap(v.Get(configSourcesKey)), factories) if err != nil { diff --git a/config/internal/configsource/parser_test.go b/config/internal/configsource/parser_test.go index b771a95b955..0600609c505 100644 --- a/config/internal/configsource/parser_test.go +++ b/config/internal/configsource/parser_test.go @@ -127,9 +127,15 @@ func (m *mockCfgSrcFactory) CreateDefaultConfig() ConfigSettings { } } -func (m *mockCfgSrcFactory) CreateConfigSource(ctx context.Context, params CreateParams, cfg ConfigSettings) (ConfigSource, error) { +func (m *mockCfgSrcFactory) CreateConfigSource(_ context.Context, _ CreateParams, cfg ConfigSettings) (ConfigSource, error) { if m.ErrOnCreateConfigSource != nil { return nil, m.ErrOnCreateConfigSource } - return &testConfigSource{}, nil + return &testConfigSource{ + ValueMap: map[string]valueEntry{ + cfg.Name(): { + Value: cfg, + }, + }, + }, nil } diff --git a/config/internal/configsource/vaultconfigsource/config.go b/config/internal/configsource/vaultconfigsource/config.go index ecd82932c9a..7669b95dfe5 100644 --- a/config/internal/configsource/vaultconfigsource/config.go +++ b/config/internal/configsource/vaultconfigsource/config.go @@ -15,16 +15,24 @@ package vaultconfigsource -import "go.opentelemetry.io/collector/config/internal/configsource" +import ( + "time" + + "go.opentelemetry.io/collector/config/internal/configsource" +) type Config struct { *configsource.Settings // Endpoint is the address of the Vault server, typically it is set via the // VAULT_ADDR environment variable for the Vault CLI. Endpoint string `mapstructure:"endpoint"` - // Path is the Vault path where the secret to be retrieved is located. - Path string `mapstructure:"path"` // Token is the token to be used to access the Vault server, typically is set // via the VAULT_TOKEN environment variable for the Vault CLI. Token string `mapstructure:"token"` + // Path is the Vault path where the secret to be retrieved is located. + Path string `mapstructure:"path"` + // PollInterval is the interval in which the config source will check for + // changes on the data on the given Vault path. This is only used for + // non-dynamic secret stores. Defaults to 1 minute if not specified. + PollInterval time.Duration `mapstructure:"poll_interval"` } diff --git a/config/internal/configsource/vaultconfigsource/config_test.go b/config/internal/configsource/vaultconfigsource/config_test.go new file mode 100644 index 00000000000..7f32f183f98 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/config_test.go @@ -0,0 +1,73 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "context" + "go.uber.org/zap" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/internal/configsource" +) + +func TestVaultLoadConfig(t *testing.T) { + fileName := path.Join("testdata", "config.yaml") + v, err := config.NewParserFromFile(fileName) + require.NoError(t, err) + + factories := map[config.Type]configsource.Factory{ + typeStr: NewFactory(), + } + + actualSettings, err := configsource.Load(context.Background(), v, factories) + require.NoError(t, err) + + expectedSettings := map[string]configsource.ConfigSettings{ + "vault": &Config{ + Settings: &configsource.Settings{ + TypeVal: "vault", + NameVal: "vault", + }, + Endpoint: "http://localhost:8200", + Token: "dev_token", + Path: "secret/kv", + PollInterval: 1 * time.Minute, + }, + "vault/poll_interval": &Config{ + Settings: &configsource.Settings{ + TypeVal: "vault", + NameVal: "vault/poll_interval", + }, + Endpoint: "https://localhost:8200", + Token: "other_token", + Path: "other/path/kv", + PollInterval: 10 * time.Second, + }, + } + + require.Equal(t, expectedSettings, actualSettings) + + params := configsource.CreateParams{ + Logger: zap.NewNop(), + } + _, err = configsource.Build(context.Background(), actualSettings, params, factories) + require.NoError(t, err) +} diff --git a/config/internal/configsource/vaultconfigsource/configsource.go b/config/internal/configsource/vaultconfigsource/configsource.go index f2ee6769b38..c0a720ac64e 100644 --- a/config/internal/configsource/vaultconfigsource/configsource.go +++ b/config/internal/configsource/vaultconfigsource/configsource.go @@ -26,31 +26,34 @@ import ( ) type vaultConfigSource struct { - client *api.Client - path string + logger *zap.Logger + client *api.Client + path string + pollInterval time.Duration } var _ configsource.ConfigSource = (*vaultConfigSource)(nil) func (v *vaultConfigSource) NewSession(context.Context) (configsource.Session, error) { - // TODO: Logger and poll interval should not be hard coded here but come from factory creating the config source. - return newSession(v.client, v.path, zap.NewNop(), 2*time.Second) + return newSession(v.client, v.path, v.logger, v.pollInterval) } -func newConfigSource(address, token, path string) (*vaultConfigSource, error) { +func newConfigSource(logger *zap.Logger, cfg *Config) (*vaultConfigSource, error) { // Client doesn't connect on creation and can't be closed. Keeping the same instance // for all sessions is ok. client, err := api.NewClient(&api.Config{ - Address: address, + Address: cfg.Endpoint, }) if err != nil { return nil, err } - client.SetToken(token) + client.SetToken(cfg.Token) return &vaultConfigSource{ - client: client, - path: path, + logger: logger, + client: client, + path: cfg.Path, + pollInterval: cfg.PollInterval, }, nil } diff --git a/config/internal/configsource/vaultconfigsource/configsource_test.go b/config/internal/configsource/vaultconfigsource/configsource_test.go index 6ce705ee78e..6a4cd264add 100644 --- a/config/internal/configsource/vaultconfigsource/configsource_test.go +++ b/config/internal/configsource/vaultconfigsource/configsource_test.go @@ -18,30 +18,34 @@ package vaultconfigsource import ( "testing" + "go.uber.org/zap" + "github.com/stretchr/testify/require" ) func TestVaultNewConfigSource(t *testing.T) { tests := []struct { name string - address string - path string - token string + config *Config wantErr bool }{ { - name: "minimal", - address: "https://some.server:1234/", + name: "minimal", + config: &Config{ + Endpoint: "https://some.server:1234/", + }, }, { - name: "invalid_address", - address: "invalid\baddress", + name: "invalid_endpoint", + config: &Config{ + Endpoint: "some\bad_endpoint", + }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfgSrc, err := newConfigSource(tt.address, tt.token, tt.path) + cfgSrc, err := newConfigSource(zap.NewNop(), tt.config) if tt.wantErr { require.Error(t, err) require.Nil(t, cfgSrc) diff --git a/config/internal/configsource/vaultconfigsource/factory.go b/config/internal/configsource/vaultconfigsource/factory.go new file mode 100644 index 00000000000..fee7f33d3f2 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/factory.go @@ -0,0 +1,85 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/internal/configsource" +) + +const ( + // The "type" of Vault config sources in configuration. + typeStr = "vault" + + defaultPollInterval = 1 * time.Minute +) + +// Private error types to help with testability. +type ( + errMissingEndpoint struct{ error } + errInvalidEndpoint struct{ error } + errMissingToken struct{ error } + errMissingPath struct{ error } + errNonPositivePollInterval struct{ error } +) + +type vaultFactory struct{} + +func (v *vaultFactory) Type() config.Type { + return typeStr +} + +func (v *vaultFactory) CreateDefaultConfig() configsource.ConfigSettings { + return &Config{ + Settings: configsource.NewSettings(typeStr), + PollInterval: defaultPollInterval, + } +} + +func (v *vaultFactory) CreateConfigSource(_ context.Context, params configsource.CreateParams, cfg configsource.ConfigSettings) (configsource.ConfigSource, error) { + vaultCfg := cfg.(*Config) + + if vaultCfg.Endpoint == "" { + return nil, &errMissingEndpoint{errors.New("cannot connect to vault with an empty endpoint")} + } + + if _, err := url.ParseRequestURI(vaultCfg.Endpoint); err != nil { + return nil, &errInvalidEndpoint{fmt.Errorf("invalid endpoint %q: %w", vaultCfg.Endpoint, err)} + } + + if vaultCfg.Token == "" { + return nil, &errMissingToken{errors.New("cannot connect to vault with an emtpy token")} + } + + if vaultCfg.Path == "" { + return nil, &errMissingPath{errors.New("cannot connect to vault with an empty path")} + } + + if vaultCfg.PollInterval <= 0 { + return nil, &errNonPositivePollInterval{errors.New("poll_interval must to be positive")} + } + + return newConfigSource(params.Logger, vaultCfg) +} + +func NewFactory() configsource.Factory { + return &vaultFactory{} +} diff --git a/config/internal/configsource/vaultconfigsource/factory_test.go b/config/internal/configsource/vaultconfigsource/factory_test.go new file mode 100644 index 00000000000..89784a418b7 --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/factory_test.go @@ -0,0 +1,99 @@ +// Copyright 2020 Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vaultconfigsource + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/internal/configsource" +) + +func TestVaultFactory_CreateConfigSource(t *testing.T) { + factory := NewFactory() + assert.Equal(t, config.Type("vault"), factory.Type()) + createParams := configsource.CreateParams{ + Logger: zap.NewNop(), + } + tests := []struct { + name string + config *Config + wantErr error + }{ + { + name: "missing_endpoint", + config: &Config{}, + wantErr: &errMissingEndpoint{}, + }, + { + name: "invalid_endpoint", + config: &Config{ + Endpoint: "some\bad/endpoint", + }, + wantErr: &errInvalidEndpoint{}, + }, + { + name: "missing_token", + config: &Config{ + Endpoint: "http://localhost:8200", + }, + wantErr: &errMissingToken{}, + }, + { + name: "missing_path", + config: &Config{ + Endpoint: "http://localhost:8200", + Token: "some_token", + }, + wantErr: &errMissingPath{}, + }, + { + name: "invalid_poll_interval", + config: &Config{ + Endpoint: "http://localhost:8200", + Token: "some_token", + Path: "some/path", + }, + wantErr: &errNonPositivePollInterval{}, + }, + { + name: "success", + config: &Config{ + Endpoint: "http://localhost:8200", + Token: "some_token", + Path: "some/path", + PollInterval: 2 * time.Minute, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := factory.CreateConfigSource(context.Background(), createParams, tt.config) + require.IsType(t, tt.wantErr, err) + if tt.wantErr == nil { + assert.NotNil(t, actual) + } else { + assert.Nil(t, actual) + } + }) + } +} diff --git a/config/internal/configsource/vaultconfigsource/session_test.go b/config/internal/configsource/vaultconfigsource/session_test.go index 75012de05e7..ee3cb79d400 100644 --- a/config/internal/configsource/vaultconfigsource/session_test.go +++ b/config/internal/configsource/vaultconfigsource/session_test.go @@ -63,7 +63,15 @@ func TestVaultSessionForKV(t *testing.T) { defer requireCmdRun(t, stopVault) requireCmdRun(t, setupKVStore) - cs, err := newConfigSource(address, token, "secret/data/kv") + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: token, + Path: "secret/data/kv", + PollInterval: 2 * time.Second, + } + + cs, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cs) @@ -99,7 +107,15 @@ func TestVaultPollingKVUpdate(t *testing.T) { defer requireCmdRun(t, stopVault) requireCmdRun(t, setupKVStore) - cs, err := newConfigSource(address, token, "secret/data/kv") + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: token, + Path: "secret/data/kv", + PollInterval: 2 * time.Second, + } + + cs, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cs) @@ -172,7 +188,15 @@ func TestVaultRenewableSecret(t *testing.T) { requireCmdRun(t, setupMongoVaultPlugin) requireCmdRun(t, setupMongoSecret) - cs, err := newConfigSource(address, token, "database/creds/my-role") + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: token, + Path: "database/creds/my-role", + PollInterval: 2 * time.Second, + } + + cs, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cs) @@ -235,7 +259,15 @@ func TestVaultV1SecretWithTTL(t *testing.T) { requireCmdRun(t, createKVVer1Store) requireCmdRun(t, setupKVVer1Store) - cs, err := newConfigSource(address, token, "kv/my-secret") + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: token, + Path: "kv/my-secret", + PollInterval: 2 * time.Second, + } + + cs, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cs) @@ -295,7 +327,15 @@ func TestVaultV1NonWatchableSecret(t *testing.T) { requireCmdRun(t, createKVVer1Store) requireCmdRun(t, setupKVVer1NoTTL) - cs, err := newConfigSource(address, token, "kv/my-secret") + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: token, + Path: "kv/my-secret", + PollInterval: 2 * time.Second, + } + + cs, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cs) @@ -371,7 +411,15 @@ func TestVaultRetrieveErrors(t *testing.T) { testToken = tt.token } - cfgSrc, err := newConfigSource(address, testToken, tt.path) + logger := zap.NewNop() + config := Config{ + Endpoint: address, + Token: testToken, + Path: tt.path, + PollInterval: 2 * time.Second, + } + + cfgSrc, err := newConfigSource(logger, &config) require.NoError(t, err) require.NotNil(t, cfgSrc) diff --git a/config/internal/configsource/vaultconfigsource/testdata/config.yaml b/config/internal/configsource/vaultconfigsource/testdata/config.yaml new file mode 100644 index 00000000000..4b0bbf0f0ee --- /dev/null +++ b/config/internal/configsource/vaultconfigsource/testdata/config.yaml @@ -0,0 +1,10 @@ +config_sources: + vault: + endpoint: http://localhost:8200 + token: dev_token + path: secret/kv + vault/poll_interval: + endpoint: https://localhost:8200 + token: other_token + path: other/path/kv + poll_interval: 10s From 65941c0d4ad944eba09803d5a0d248f515d6e78f Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 8 Apr 2021 23:33:55 -0700 Subject: [PATCH 06/10] Fixes from goimp, lint, etc --- config/internal/configsource/vaultconfigsource/config_test.go | 2 +- .../configsource/vaultconfigsource/configsource_test.go | 3 +-- config/internal/configsource/vaultconfigsource/factory.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/internal/configsource/vaultconfigsource/config_test.go b/config/internal/configsource/vaultconfigsource/config_test.go index 7f32f183f98..55b203d73b0 100644 --- a/config/internal/configsource/vaultconfigsource/config_test.go +++ b/config/internal/configsource/vaultconfigsource/config_test.go @@ -17,12 +17,12 @@ package vaultconfigsource import ( "context" - "go.uber.org/zap" "path" "testing" "time" "github.com/stretchr/testify/require" + "go.uber.org/zap" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/internal/configsource" diff --git a/config/internal/configsource/vaultconfigsource/configsource_test.go b/config/internal/configsource/vaultconfigsource/configsource_test.go index 6a4cd264add..2b43f22258e 100644 --- a/config/internal/configsource/vaultconfigsource/configsource_test.go +++ b/config/internal/configsource/vaultconfigsource/configsource_test.go @@ -18,9 +18,8 @@ package vaultconfigsource import ( "testing" - "go.uber.org/zap" - "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestVaultNewConfigSource(t *testing.T) { diff --git a/config/internal/configsource/vaultconfigsource/factory.go b/config/internal/configsource/vaultconfigsource/factory.go index fee7f33d3f2..63291248e19 100644 --- a/config/internal/configsource/vaultconfigsource/factory.go +++ b/config/internal/configsource/vaultconfigsource/factory.go @@ -66,7 +66,7 @@ func (v *vaultFactory) CreateConfigSource(_ context.Context, params configsource } if vaultCfg.Token == "" { - return nil, &errMissingToken{errors.New("cannot connect to vault with an emtpy token")} + return nil, &errMissingToken{errors.New("cannot connect to vault with an empty token")} } if vaultCfg.Path == "" { From d0a755bb56322ef45b1c8b779fd36622f63eeaa0 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Sun, 11 Apr 2021 11:10:14 -0700 Subject: [PATCH 07/10] Move configsource to experimental --- {config/internal => experimental}/configsource/builder.go | 0 {config/internal => experimental}/configsource/builder_test.go | 0 {config/internal => experimental}/configsource/component.go | 0 {config/internal => experimental}/configsource/factory.go | 0 {config/internal => experimental}/configsource/manager.go | 0 {config/internal => experimental}/configsource/manager_test.go | 0 {config/internal => experimental}/configsource/parser.go | 0 {config/internal => experimental}/configsource/parser_test.go | 0 {config/internal => experimental}/configsource/settings.go | 0 .../configsource/testdata/arrays_and_maps.yaml | 0 .../configsource/testdata/arrays_and_maps_expected.yaml | 0 .../internal => experimental}/configsource/testdata/bad_name.yaml | 0 .../configsource/testdata/basic_config.yaml | 0 .../configsource/testdata/duplicated_name.yaml | 0 .../configsource/testdata/envvar_cfgsrc_mix.yaml | 0 .../configsource/testdata/envvar_cfgsrc_mix_expected.yaml | 0 .../configsource/testdata/params_handling.yaml | 0 .../configsource/testdata/params_handling_expected.yaml | 0 .../configsource/testdata/unknown_field.yaml | 0 .../configsource/vaultconfigsource/configsource_test.go | 0 .../configsource/vaultconfigsource/testdata/config.yaml | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename {config/internal => experimental}/configsource/builder.go (100%) rename {config/internal => experimental}/configsource/builder_test.go (100%) rename {config/internal => experimental}/configsource/component.go (100%) rename {config/internal => experimental}/configsource/factory.go (100%) rename {config/internal => experimental}/configsource/manager.go (100%) rename {config/internal => experimental}/configsource/manager_test.go (100%) rename {config/internal => experimental}/configsource/parser.go (100%) rename {config/internal => experimental}/configsource/parser_test.go (100%) rename {config/internal => experimental}/configsource/settings.go (100%) rename {config/internal => experimental}/configsource/testdata/arrays_and_maps.yaml (100%) rename {config/internal => experimental}/configsource/testdata/arrays_and_maps_expected.yaml (100%) rename {config/internal => experimental}/configsource/testdata/bad_name.yaml (100%) rename {config/internal => experimental}/configsource/testdata/basic_config.yaml (100%) rename {config/internal => experimental}/configsource/testdata/duplicated_name.yaml (100%) rename {config/internal => experimental}/configsource/testdata/envvar_cfgsrc_mix.yaml (100%) rename {config/internal => experimental}/configsource/testdata/envvar_cfgsrc_mix_expected.yaml (100%) rename {config/internal => experimental}/configsource/testdata/params_handling.yaml (100%) rename {config/internal => experimental}/configsource/testdata/params_handling_expected.yaml (100%) rename {config/internal => experimental}/configsource/testdata/unknown_field.yaml (100%) rename {config/internal => experimental}/configsource/vaultconfigsource/configsource_test.go (100%) rename {config/internal => experimental}/configsource/vaultconfigsource/testdata/config.yaml (100%) diff --git a/config/internal/configsource/builder.go b/experimental/configsource/builder.go similarity index 100% rename from config/internal/configsource/builder.go rename to experimental/configsource/builder.go diff --git a/config/internal/configsource/builder_test.go b/experimental/configsource/builder_test.go similarity index 100% rename from config/internal/configsource/builder_test.go rename to experimental/configsource/builder_test.go diff --git a/config/internal/configsource/component.go b/experimental/configsource/component.go similarity index 100% rename from config/internal/configsource/component.go rename to experimental/configsource/component.go diff --git a/config/internal/configsource/factory.go b/experimental/configsource/factory.go similarity index 100% rename from config/internal/configsource/factory.go rename to experimental/configsource/factory.go diff --git a/config/internal/configsource/manager.go b/experimental/configsource/manager.go similarity index 100% rename from config/internal/configsource/manager.go rename to experimental/configsource/manager.go diff --git a/config/internal/configsource/manager_test.go b/experimental/configsource/manager_test.go similarity index 100% rename from config/internal/configsource/manager_test.go rename to experimental/configsource/manager_test.go diff --git a/config/internal/configsource/parser.go b/experimental/configsource/parser.go similarity index 100% rename from config/internal/configsource/parser.go rename to experimental/configsource/parser.go diff --git a/config/internal/configsource/parser_test.go b/experimental/configsource/parser_test.go similarity index 100% rename from config/internal/configsource/parser_test.go rename to experimental/configsource/parser_test.go diff --git a/config/internal/configsource/settings.go b/experimental/configsource/settings.go similarity index 100% rename from config/internal/configsource/settings.go rename to experimental/configsource/settings.go diff --git a/config/internal/configsource/testdata/arrays_and_maps.yaml b/experimental/configsource/testdata/arrays_and_maps.yaml similarity index 100% rename from config/internal/configsource/testdata/arrays_and_maps.yaml rename to experimental/configsource/testdata/arrays_and_maps.yaml diff --git a/config/internal/configsource/testdata/arrays_and_maps_expected.yaml b/experimental/configsource/testdata/arrays_and_maps_expected.yaml similarity index 100% rename from config/internal/configsource/testdata/arrays_and_maps_expected.yaml rename to experimental/configsource/testdata/arrays_and_maps_expected.yaml diff --git a/config/internal/configsource/testdata/bad_name.yaml b/experimental/configsource/testdata/bad_name.yaml similarity index 100% rename from config/internal/configsource/testdata/bad_name.yaml rename to experimental/configsource/testdata/bad_name.yaml diff --git a/config/internal/configsource/testdata/basic_config.yaml b/experimental/configsource/testdata/basic_config.yaml similarity index 100% rename from config/internal/configsource/testdata/basic_config.yaml rename to experimental/configsource/testdata/basic_config.yaml diff --git a/config/internal/configsource/testdata/duplicated_name.yaml b/experimental/configsource/testdata/duplicated_name.yaml similarity index 100% rename from config/internal/configsource/testdata/duplicated_name.yaml rename to experimental/configsource/testdata/duplicated_name.yaml diff --git a/config/internal/configsource/testdata/envvar_cfgsrc_mix.yaml b/experimental/configsource/testdata/envvar_cfgsrc_mix.yaml similarity index 100% rename from config/internal/configsource/testdata/envvar_cfgsrc_mix.yaml rename to experimental/configsource/testdata/envvar_cfgsrc_mix.yaml diff --git a/config/internal/configsource/testdata/envvar_cfgsrc_mix_expected.yaml b/experimental/configsource/testdata/envvar_cfgsrc_mix_expected.yaml similarity index 100% rename from config/internal/configsource/testdata/envvar_cfgsrc_mix_expected.yaml rename to experimental/configsource/testdata/envvar_cfgsrc_mix_expected.yaml diff --git a/config/internal/configsource/testdata/params_handling.yaml b/experimental/configsource/testdata/params_handling.yaml similarity index 100% rename from config/internal/configsource/testdata/params_handling.yaml rename to experimental/configsource/testdata/params_handling.yaml diff --git a/config/internal/configsource/testdata/params_handling_expected.yaml b/experimental/configsource/testdata/params_handling_expected.yaml similarity index 100% rename from config/internal/configsource/testdata/params_handling_expected.yaml rename to experimental/configsource/testdata/params_handling_expected.yaml diff --git a/config/internal/configsource/testdata/unknown_field.yaml b/experimental/configsource/testdata/unknown_field.yaml similarity index 100% rename from config/internal/configsource/testdata/unknown_field.yaml rename to experimental/configsource/testdata/unknown_field.yaml diff --git a/config/internal/configsource/vaultconfigsource/configsource_test.go b/experimental/configsource/vaultconfigsource/configsource_test.go similarity index 100% rename from config/internal/configsource/vaultconfigsource/configsource_test.go rename to experimental/configsource/vaultconfigsource/configsource_test.go diff --git a/config/internal/configsource/vaultconfigsource/testdata/config.yaml b/experimental/configsource/vaultconfigsource/testdata/config.yaml similarity index 100% rename from config/internal/configsource/vaultconfigsource/testdata/config.yaml rename to experimental/configsource/vaultconfigsource/testdata/config.yaml From 6d45d846655a43d620d26619b6a0190befd1e15d Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Sun, 11 Apr 2021 11:10:48 -0700 Subject: [PATCH 08/10] Move configsource to experimental --- .../configsource/vaultconfigsource/config.go | 2 +- .../configsource/vaultconfigsource/config_test.go | 2 +- .../configsource/vaultconfigsource/configsource.go | 2 +- .../configsource/vaultconfigsource/factory.go | 2 +- .../configsource/vaultconfigsource/factory_test.go | 2 +- .../configsource/vaultconfigsource/session.go | 2 +- .../configsource/vaultconfigsource/session_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename {config/internal => experimental}/configsource/vaultconfigsource/config.go (95%) rename {config/internal => experimental}/configsource/vaultconfigsource/config_test.go (96%) rename {config/internal => experimental}/configsource/vaultconfigsource/configsource.go (96%) rename {config/internal => experimental}/configsource/vaultconfigsource/factory.go (97%) rename {config/internal => experimental}/configsource/vaultconfigsource/factory_test.go (97%) rename {config/internal => experimental}/configsource/vaultconfigsource/session.go (99%) rename {config/internal => experimental}/configsource/vaultconfigsource/session_test.go (99%) diff --git a/config/internal/configsource/vaultconfigsource/config.go b/experimental/configsource/vaultconfigsource/config.go similarity index 95% rename from config/internal/configsource/vaultconfigsource/config.go rename to experimental/configsource/vaultconfigsource/config.go index 7669b95dfe5..896b1ca11e3 100644 --- a/config/internal/configsource/vaultconfigsource/config.go +++ b/experimental/configsource/vaultconfigsource/config.go @@ -18,7 +18,7 @@ package vaultconfigsource import ( "time" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) type Config struct { diff --git a/config/internal/configsource/vaultconfigsource/config_test.go b/experimental/configsource/vaultconfigsource/config_test.go similarity index 96% rename from config/internal/configsource/vaultconfigsource/config_test.go rename to experimental/configsource/vaultconfigsource/config_test.go index 55b203d73b0..02874dc232f 100644 --- a/config/internal/configsource/vaultconfigsource/config_test.go +++ b/experimental/configsource/vaultconfigsource/config_test.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) func TestVaultLoadConfig(t *testing.T) { diff --git a/config/internal/configsource/vaultconfigsource/configsource.go b/experimental/configsource/vaultconfigsource/configsource.go similarity index 96% rename from config/internal/configsource/vaultconfigsource/configsource.go rename to experimental/configsource/vaultconfigsource/configsource.go index c0a720ac64e..91f79ef7ccc 100644 --- a/config/internal/configsource/vaultconfigsource/configsource.go +++ b/experimental/configsource/vaultconfigsource/configsource.go @@ -22,7 +22,7 @@ import ( "github.com/hashicorp/vault/api" "go.uber.org/zap" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) type vaultConfigSource struct { diff --git a/config/internal/configsource/vaultconfigsource/factory.go b/experimental/configsource/vaultconfigsource/factory.go similarity index 97% rename from config/internal/configsource/vaultconfigsource/factory.go rename to experimental/configsource/vaultconfigsource/factory.go index 63291248e19..8599fb80eec 100644 --- a/config/internal/configsource/vaultconfigsource/factory.go +++ b/experimental/configsource/vaultconfigsource/factory.go @@ -22,7 +22,7 @@ import ( "time" "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) const ( diff --git a/config/internal/configsource/vaultconfigsource/factory_test.go b/experimental/configsource/vaultconfigsource/factory_test.go similarity index 97% rename from config/internal/configsource/vaultconfigsource/factory_test.go rename to experimental/configsource/vaultconfigsource/factory_test.go index 89784a418b7..eec074100fc 100644 --- a/config/internal/configsource/vaultconfigsource/factory_test.go +++ b/experimental/configsource/vaultconfigsource/factory_test.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) func TestVaultFactory_CreateConfigSource(t *testing.T) { diff --git a/config/internal/configsource/vaultconfigsource/session.go b/experimental/configsource/vaultconfigsource/session.go similarity index 99% rename from config/internal/configsource/vaultconfigsource/session.go rename to experimental/configsource/vaultconfigsource/session.go index a4756eaee79..701372d3d74 100644 --- a/config/internal/configsource/vaultconfigsource/session.go +++ b/experimental/configsource/vaultconfigsource/session.go @@ -26,7 +26,7 @@ import ( "github.com/hashicorp/vault/api" "go.uber.org/zap" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) var errInvalidPollInterval = errors.New("poll interval must be greater than zero") diff --git a/config/internal/configsource/vaultconfigsource/session_test.go b/experimental/configsource/vaultconfigsource/session_test.go similarity index 99% rename from config/internal/configsource/vaultconfigsource/session_test.go rename to experimental/configsource/vaultconfigsource/session_test.go index ee3cb79d400..a114de4211c 100644 --- a/config/internal/configsource/vaultconfigsource/session_test.go +++ b/experimental/configsource/vaultconfigsource/session_test.go @@ -30,7 +30,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" - "go.opentelemetry.io/collector/config/internal/configsource" + "go.opentelemetry.io/collector/experimental/configsource" ) const ( From 5c91d09147054df10771ba29e7f79cc30c1bb5bc Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Sun, 11 Apr 2021 11:25:23 -0700 Subject: [PATCH 09/10] Add missing comments --- experimental/configsource/factory.go | 1 + experimental/configsource/settings.go | 2 ++ experimental/configsource/vaultconfigsource/config.go | 1 + experimental/configsource/vaultconfigsource/factory.go | 1 + 4 files changed, 5 insertions(+) diff --git a/experimental/configsource/factory.go b/experimental/configsource/factory.go index b5b605b23cf..e4c0236701c 100644 --- a/experimental/configsource/factory.go +++ b/experimental/configsource/factory.go @@ -50,4 +50,5 @@ type Factory interface { CreateConfigSource(ctx context.Context, params CreateParams, cfg ConfigSettings) (ConfigSource, error) } +// Factories is a map from types of ConfigSource and the respective Factory for the type. type Factories map[config.Type]Factory diff --git a/experimental/configsource/settings.go b/experimental/configsource/settings.go index 5057f3f76b5..05a6cff155f 100644 --- a/experimental/configsource/settings.go +++ b/experimental/configsource/settings.go @@ -18,6 +18,8 @@ import ( "go.opentelemetry.io/collector/config" ) +// ConfigSettings is an interface that must be supported by all configuration objects +// of ConfigSource implementations. type ConfigSettings interface { config.NamedEntity } diff --git a/experimental/configsource/vaultconfigsource/config.go b/experimental/configsource/vaultconfigsource/config.go index 896b1ca11e3..e4c29f8f854 100644 --- a/experimental/configsource/vaultconfigsource/config.go +++ b/experimental/configsource/vaultconfigsource/config.go @@ -21,6 +21,7 @@ import ( "go.opentelemetry.io/collector/experimental/configsource" ) +// Config holds the configuration settings for Vault ConfigSource objects. type Config struct { *configsource.Settings // Endpoint is the address of the Vault server, typically it is set via the diff --git a/experimental/configsource/vaultconfigsource/factory.go b/experimental/configsource/vaultconfigsource/factory.go index 8599fb80eec..c620ad305a7 100644 --- a/experimental/configsource/vaultconfigsource/factory.go +++ b/experimental/configsource/vaultconfigsource/factory.go @@ -80,6 +80,7 @@ func (v *vaultFactory) CreateConfigSource(_ context.Context, params configsource return newConfigSource(params.Logger, vaultCfg) } +// NewFactory creates a factory for Vault ConfigSource objects. func NewFactory() configsource.Factory { return &vaultFactory{} } From 6a9bdcc09e2ebf11926ea4180b2d046b17b35175 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 23:30:55 +0000 Subject: [PATCH 10/10] Bump github.com/shirou/gopsutil Bumps [github.com/shirou/gopsutil](https://github.com/shirou/gopsutil) from 3.21.3+incompatible to 3.21.11+incompatible. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/compare/v3.21.3...v3.21.11) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 5 ++--- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index df8ad4b3398..517ff4885ca 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.14 require ( contrib.go.opencensus.io/exporter/prometheus v0.3.0 github.com/Shopify/sarama v1.28.0 - github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect github.com/antonmedv/expr v1.8.9 github.com/apache/thrift v0.13.0 github.com/cenkalti/backoff/v4 v4.1.0 @@ -13,7 +12,6 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/fatih/structtag v1.2.0 github.com/go-kit/kit v0.10.0 - github.com/go-ole/go-ole v1.2.5 // indirect github.com/gogo/protobuf v1.3.2 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.5.2 @@ -32,7 +30,7 @@ require ( github.com/prometheus/common v0.20.0 github.com/prometheus/prometheus v1.8.2-0.20210217141258-a6be548dbc17 github.com/rs/cors v1.7.0 - github.com/shirou/gopsutil v3.21.3+incompatible + github.com/shirou/gopsutil v3.21.11+incompatible github.com/soheilhy/cmux v0.1.5 github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.1.3 @@ -42,6 +40,7 @@ require ( github.com/tklauser/go-sysconf v0.3.4 // indirect github.com/uber/jaeger-lib v2.4.1+incompatible github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 go.uber.org/atomic v1.7.0 go.uber.org/zap v1.16.0 diff --git a/go.sum b/go.sum index 67b3bfc0208..c719e2cae65 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/Shopify/sarama v1.28.0 h1:lOi3SfE6OcFlW9Trgtked2aHNZ2BIG/d6Do+PEUAqqM github.com/Shopify/sarama v1.28.0/go.mod h1:j/2xTrU39dlzBmsxF1eQ2/DdWrxyBCl6pzz7a81o/ZY= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= -github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -290,8 +288,8 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -938,8 +936,8 @@ github.com/securego/gosec v0.0.0-20200203094520-d13bb6d2420c/go.mod h1:gp0gaHj0W github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= -github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= @@ -1040,6 +1038,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=