From 228e514b7b7673bf36cf89ad007a44664e52a272 Mon Sep 17 00:00:00 2001 From: ejlee-intel Date: Wed, 25 Jan 2023 10:08:33 -0700 Subject: [PATCH] feat: Push common config to config provider (#4306) * feat: Push common config to config provider Signed-off-by: Elizabeth J Lee --- .../Dockerfile | 2 +- go.mod | 4 +- go.sum | 4 +- internal/core/common_config/main.go | 121 +++++++++++++++++- 4 files changed, 121 insertions(+), 10 deletions(-) diff --git a/cmd/core-common-config-bootstrapper/Dockerfile b/cmd/core-common-config-bootstrapper/Dockerfile index 28d0ab2784..e0280535b9 100644 --- a/cmd/core-common-config-bootstrapper/Dockerfile +++ b/cmd/core-common-config-bootstrapper/Dockerfile @@ -49,4 +49,4 @@ RUN chmod 755 /usr/local/bin/entrypoint.sh \ && ln -s /usr/local/bin/entrypoint.sh / ENTRYPOINT ["entrypoint.sh"] -CMD ["/core-common-config-bootstrapper", "-cp=consul.http://edgex-core-consul:8500"] +CMD ["/core-common-config-bootstrapper", "-cp=consul.http://edgex-core-consul:8500", "-cf=configuration.yaml"] diff --git a/go.mod b/go.mod index 09c839d82a..88bd22d569 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/edgexfoundry/edgex-go require ( bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690 github.com/eclipse/paho.mqtt.golang v1.4.2 - github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12 + github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16 + github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.6 github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.4 github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.5 @@ -27,7 +28,6 @@ require ( github.com/armon/go-metrics v0.3.10 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2 // indirect github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.3 // indirect github.com/fatih/color v1.9.0 // indirect github.com/go-kit/log v0.2.1 // indirect diff --git a/go.sum b/go.sum index 702c549fff..b4c149014b 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eclipse/paho.mqtt.golang v1.4.2 h1:66wOzfUHSSI1zamx7jR6yMEI5EuHnT1G6rNA5PM12m4= github.com/eclipse/paho.mqtt.golang v1.4.2/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA= -github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12 h1:kLllnz4O5QRa2SPALhEIanzL20DtczsJYL5+NGKNz3o= -github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12/go.mod h1:JqdvZ2lSGydoHC++jlvfeawlB5ogFRCmWeYkoW+HHMo= +github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16 h1:MN6dOZHbYkW8JRDlEin/7T9nInOwpikZhthH0QaMksQ= +github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16/go.mod h1:G2C3aUWZ96nZU03XRvCBntU13eUjXGIB9KqRKrhI8h8= github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2 h1:xp5MsP+qf/fuJxy8fT7k1N+c4j4C6w04qMCBXm6id7o= github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2/go.mod h1:1Vv4uWAo6r7k6jUlqVJW8JOL6YKVBc6sRL8Al3DrMck= github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.6 h1:RQFs/HjVOi1X3YxJ8sm4vuX8nhKgH0caSf9RtjQvdeI= diff --git a/internal/core/common_config/main.go b/internal/core/common_config/main.go index 93ff93754e..83103aa663 100644 --- a/internal/core/common_config/main.go +++ b/internal/core/common_config/main.go @@ -16,21 +16,30 @@ package common_config import ( "context" + "fmt" + "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/config" "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/container" "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/environment" "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/flags" "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/secret" "github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/startup" "github.com/edgexfoundry/go-mod-bootstrap/v3/di" + "github.com/edgexfoundry/go-mod-configuration/v3/configuration" "github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger" "github.com/edgexfoundry/go-mod-core-contracts/v3/common" "github.com/edgexfoundry/go-mod-core-contracts/v3/models" + "gopkg.in/yaml.v3" "os" "os/signal" + "sort" "sync" "syscall" ) +const ( + commonConfigDone = "IsCommonConfigReady" +) + func Main(ctx context.Context, cancel context.CancelFunc) { // All common command-line flags have been moved to DefaultCommonFlags. Service specific flags can be add here, @@ -56,7 +65,6 @@ func Main(ctx context.Context, cancel context.CancelFunc) { return lc }, }) - secretProvider, err := secret.NewSecretProvider(nil, environment.NewVariables(lc), ctx, startupTimer, dic, common.CoreCommonConfigServiceKey) if err != nil { lc.Errorf("failed to create Secret Provider: %v", err) @@ -65,14 +73,47 @@ func Main(ctx context.Context, cancel context.CancelFunc) { lc.Info("Secret Provider created") - _, err = secretProvider.GetAccessToken("consul", common.CoreCommonConfigServiceKey) + // need to use in-line function to set the callback type for getAccessToken used in CreateProviderClient to allow + // access to the config provider in secure mode + getAccessToken := func() (string, error) { + accessToken, err := secretProvider.GetAccessToken("consul", common.CoreCommonConfigServiceKey) + if err != nil { + return "", fmt.Errorf("failed to get Configuration Provider access token: %s", err.Error()) + } + lc.Infof("Got Config Provider Access Token with length %d", len(accessToken)) + return accessToken, err + } + + // create config client + envVars := environment.NewVariables(lc) + configProviderInfo, err := config.NewProviderInfo(envVars, f.ConfigProviderUrl()) + if err != nil { + lc.Errorf("failed to get Provider Info for the common configuration: %s", err.Error()) + os.Exit(1) + } + configClient, err := config.CreateProviderClient(lc, common.CoreCommonConfigServiceKey, common.ConfigStemCore, getAccessToken, configProviderInfo.ServiceConfig()) + if err != nil { + lc.Errorf("failed to create provider client for the common configuration: %s", err.Error()) + os.Exit(1) + } + // check to see if the configuration exists in the config provider + hasConfig, err := configClient.HasConfiguration() if err != nil { - lc.Errorf("failed to get Access Token for config provider: %v", err) + lc.Errorf("failed to determine if common configuration exists in the provider: %s", err.Error()) os.Exit(1) } + lc.Infof("configuration exists: %v", hasConfig) + // load the yaml file and push it using the config client + if !hasConfig || f.OverwriteConfig() { + yamlFile := config.GetConfigLocation(lc, f) + lc.Infof("parsing %s for configuration", yamlFile) + err = loadYaml(lc, yamlFile, configClient) + if err != nil { + lc.Error(err.Error()) + os.Exit(1) + } + } - lc.Info("Got Config Provider Access Token") - lc.Info("Core Common Config Ready for stage two") lc.Info("Core Common Config exiting") os.Exit(0) } @@ -99,3 +140,73 @@ func translateInterruptToCancel(ctx context.Context, wg *sync.WaitGroup, cancel } }() } + +func loadYaml(lc logger.LoggingClient, yamlFile string, configClient configuration.Client) error { + // push not done flag to configClient + err := configClient.PutConfigurationValue(commonConfigDone, []byte("false")) + if err != nil { + return fmt.Errorf("failed to push %s on startup: %s", commonConfigDone, err.Error()) + } + lc.Infof("reading %s", yamlFile) + contents, err := os.ReadFile(yamlFile) + if err != nil { + return fmt.Errorf("failed to read common configuration file %s: %s", yamlFile, err.Error()) + } + + data := make(map[string]interface{}) + kv := make(map[string]interface{}) + + err = yaml.Unmarshal(contents, &data) + if err != nil { + return fmt.Errorf("failed to unmarshall common configuration file %s: %s", yamlFile, err.Error()) + } + + kv = buildKeyValues(data, kv, "") + + keys := make([]string, 0, len(kv)) + + for k := range kv { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, k := range keys { + v := kv[k] + err = configClient.PutConfigurationValue(k, []byte(fmt.Sprint(v))) + if err != nil { + return fmt.Errorf("failed to push common configuration key %s with value %v: %s", k, v, err.Error()) + } + } + + // push done flag to config client + err = configClient.PutConfigurationValue(commonConfigDone, []byte("true")) + if err != nil { + return fmt.Errorf("failed to push %s on completion: %s", commonConfigDone, err.Error()) + } + return nil +} + +// buildKeyValues is a helper function to parse the configuration yaml file contents +func buildKeyValues(data map[string]interface{}, kv map[string]interface{}, origKey string) map[string]interface{} { + key := origKey + for k, v := range data { + if len(key) == 0 { + key = fmt.Sprint(k) + } else { + key = fmt.Sprintf("%s/%s", key, k) + } + + vdata, ok := v.(map[string]interface{}) + if !ok { + kv[key] = v + key = origKey + continue + } + + kv = buildKeyValues(vdata, kv, key) + key = origKey + } + + return kv +}