From cac684f905ea1be2cb509bd4f27bfe4e2e2d020c Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Thu, 10 Mar 2022 11:48:05 -0500 Subject: [PATCH 01/10] WIP implementation of a Consul configsourcer plugin --- builtin/consul/config_sourcer.go | 410 +++++++++++++++++++++++++++++++ builtin/consul/consul.go | 11 + internal/plugin/plugin.go | 5 + 3 files changed, 426 insertions(+) create mode 100644 builtin/consul/config_sourcer.go create mode 100644 builtin/consul/consul.go diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go new file mode 100644 index 00000000000..314b9145941 --- /dev/null +++ b/builtin/consul/config_sourcer.go @@ -0,0 +1,410 @@ +package consul + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint-plugin-sdk/component" + "github.com/hashicorp/waypoint-plugin-sdk/docs" + pb "github.com/hashicorp/waypoint-plugin-sdk/proto/gen" + "github.com/mitchellh/mapstructure" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // retryInterval is the base retry value + retryInterval = 5 * time.Second + + // maximum back off time, this is to prevent + // exponential runaway + maxBackoffTime = 180 * time.Second +) + +type reqConfig struct { + Key string `hcl:"key,attr"` // kv path to retrieve + Namespace string `hcl:"namespace,optional"` // namespace the kv data resides within + Partition string `hcl:"partition,optional"` // partition the kv data resides within + Datacenter string `hcl:"datacenter,optional"` // datacenter the kv data resides within + AllowStale bool `hcl:"allow_stale,optional"` // whether to perform stale queries against non-leader servers +} + +func (r *reqConfig) cacheKey() string { + return fmt.Sprintf("%s/%s/%s/%s", r.Datacenter, r.Partition, r.Namespace, r.Key) +} + +type cachedVal struct { + kvPair *api.KVPair + cancel context.CancelFunc + err error +} + +type ConfigSourcerConfig struct { + // Configuration for where to talk to Consul + Address string `hcl:"address,optional"` + Scheme string `hcl:"scheme,optional"` + HTTPAuth ConsulHTTPAuth `hcl:"http_auth,optional"` + Token string `hcl:"token,optional"` + TokenFile string `hcl:"token_file,optional"` + TLSConfig TLSConfig `hcl:"tls,optional"` + + // Default location of KV data + Datacenter string `hcl:"datacenter,optional"` + Namespace string `hcl:"namespace,optional"` + Partition string `hcl:"partition,optional"` +} + +func (conf *ConfigSourcerConfig) toAPIClient() (*api.Client, error) { + apiConfig := api.Config{ + Address: conf.Address, + Scheme: conf.Scheme, + Datacenter: conf.Datacenter, + HttpAuth: conf.HTTPAuth.toApiAuth(), + Token: conf.Token, + TokenFile: conf.TokenFile, + Namespace: conf.Namespace, + Partition: conf.Partition, + TLSConfig: *conf.TLSConfig.toAPITLSConfig(), + } + + return api.NewClient(&apiConfig) +} + +type TLSConfig struct { + ServerName string `hcl:"server_name,optional"` + CAFile string `hcl:"ca_file,optional"` + CAPath string `hcl:"ca_path,optional"` + CAPem []byte `hcl:"ca_pem,optional"` + CertFile string `hcl:"cert_file,optional"` + CertPEM []byte `hcl:"cert_pem,optional"` + KeyFile string `hcl:"key_file,optional"` + KeyPEM []byte `hcl:"key_pem,optional"` + InsecureHTTPs bool `hcl:"insecure_https,optional"` +} + +func (t *TLSConfig) toAPITLSConfig() *api.TLSConfig { + return &api.TLSConfig{ + Address: t.ServerName, + CAFile: t.CAFile, + CAPath: t.CAPath, + CAPem: t.CAPem, + CertFile: t.CertFile, + CertPEM: t.CertPEM, + KeyFile: t.KeyFile, + KeyPEM: t.KeyPEM, + InsecureSkipVerify: t.InsecureHTTPs, + } +} + +type ConsulHTTPAuth struct { + Username string `hcl:"username,optional"` + Password string `hcl:"password,optional"` +} + +func (a *ConsulHTTPAuth) toApiAuth() *api.HttpBasicAuth { + if a.Username == "" && a.Password == "" { + return nil + } + + return &api.HttpBasicAuth{ + Username: a.Username, + Password: a.Password, + } +} + +type ConfigSourcer struct { + config ConfigSourcerConfig + + mu sync.Mutex + client *api.KV + cache map[string]*cachedVal +} + +// Implement Configurable +func (cs *ConfigSourcer) Config() (interface{}, error) { + return &cs.config, nil +} + +// Implement ConfigurableNotify +func (cs *ConfigSourcer) ConfigSet(config interface{}) error { + conf, ok := config.(*ConfigSourcerConfig) + if !ok { + // The Waypoint SDK should ensure this never gets hit + return fmt.Errorf("Expected *ConfigSourcerConfig as parameter") + } + + // attempt to create API client + client, err := conf.toAPIClient() + if err != nil { + return fmt.Errorf("Invalid configuration: %w", err) + } + + cs.client = client.KV() + + return nil +} + +// ReadFunc returns the function for reading configuration. +// +// The returned function can start a background goroutine to more efficiently +// watch for changes. The entrypoint will periodically call Read to check for +// updates. +// +// If the configuration changes for any dynamic configuration variable, +// the entrypoint will call Stop followed by Read, so plugins DO NOT need +// to implement config diffing. Plugins may safely assume if Read is called +// after a Stop that the config is new, and that subsequent calls have the +// same config. +// +// Read is called for ALL defined configuration variables for this source. +// If ANY change, Stop is called followed by Read again. Only one sourcer +// is active for a set of configs. +func (cs *ConfigSourcer) ReadFunc() interface{} { + return cs.read +} + +// StopFunc returns a function for stopping configuration sourcing. +// You can return nil if stopping is not necessary or supported for +// this sourcer. +// +// The stop function should stop any background processes started with Read. +func (cs *ConfigSourcer) StopFunc() interface{} { + return cs.stop +} + +func (cs *ConfigSourcer) read(ctx context.Context, log hclog.Logger, reqs []*component.ConfigRequest) ([]*pb.ConfigSource_Value, error) { + log.Trace("Reading KV data from Consul") + // Setup our lock + cs.mu.Lock() + defer cs.mu.Unlock() + + // Create our cache if this is our first time + if cs.cache == nil { + cs.cache = make(map[string]*cachedVal) + } + + // Create our Consul API client if this is our first time + if cs.client == nil { + client, err := cs.config.toAPIClient() + if err != nil { + return nil, fmt.Errorf("Invalid Consul client configuration: %w", err) + } + cs.client = client.KV() + } + + var results []*pb.ConfigSource_Value + for _, req := range reqs { + result := &pb.ConfigSource_Value{Name: req.Name} + results = append(results, result) + + // Decode our configuration + kvReq := reqConfig{ + // default to allowing stale queries + AllowStale: true, + } + if err := mapstructure.WeakDecode(req.Config, &kvReq); err != nil { + result.Result = &pb.ConfigSource_Value_Error{ + Error: status.New(codes.Aborted, err.Error()).Proto(), + } + + continue + } + + opts := &api.QueryOptions{ + Namespace: kvReq.Namespace, + Partition: kvReq.Partition, + Datacenter: kvReq.Datacenter, + AllowStale: kvReq.AllowStale, + } + reqLogger := log.With("key", kvReq.Key, "stale", kvReq.AllowStale) + if kvReq.Namespace != "" { + reqLogger = reqLogger.With("namespace", kvReq.Namespace) + } + if kvReq.Partition != "" { + reqLogger = reqLogger.With("partition", kvReq.Partition) + } + if kvReq.Datacenter != "" { + reqLogger = reqLogger.With("datacenter", kvReq.Partition) + } + + cacheVal, ok := cs.cache[kvReq.cacheKey()] + if !ok { + reqLogger.Trace("querying Consul KV") + kvpair, meta, err := cs.client.Get(kvReq.Key, opts.WithContext(ctx)) + if err != nil { + result.Result = &pb.ConfigSource_Value_Error{ + Error: status.New(codes.Aborted, err.Error()).Proto(), + } + + continue + } + + refreshCtx, cancel := context.WithCancel(context.Background()) + cacheVal = &cachedVal{ + kvPair: kvpair, + cancel: cancel, + err: nil, + } + cs.cache[kvReq.cacheKey()] = cacheVal + cs.startBlockingQuery(refreshCtx, log, kvReq, meta.LastIndex, opts) + } + + if cacheVal.err != nil { + result.Result = &pb.ConfigSource_Value_Error{ + Error: status.New(codes.Aborted, cacheVal.err.Error()).Proto(), + } + } + + if cacheVal.kvPair == nil { + result.Result = &pb.ConfigSource_Value_Error{ + Error: status.New(codes.NotFound, fmt.Sprintf("Configuration for key %s doesn't exist", kvReq.Key)).Proto(), + } + + continue + } + + result.Result = &pb.ConfigSource_Value_Value{ + Value: string(cacheVal.kvPair.Value), + } + } + + return results, nil +} + +func (cs *ConfigSourcer) startBlockingQuery(ctx context.Context, logger hclog.Logger, kvReq reqConfig, lastIndex uint64, opts *api.QueryOptions) { + go func() { + opts := opts.WithContext(ctx) + failures := 0 + // Ideally we would use the github.com/hashicorp/consul/api/watch package. However that package doesn't support + // namespaces and partitions except with the global client defaults and thus isn't suitable for this usage. + + for { + logger.Trace("Issuing blocking query", "wait-index", lastIndex) + // set the wait index to use for the query + opts.WaitIndex = lastIndex + pair, meta, err := cs.client.Get(kvReq.Key, opts) + + // check if we are being stopped + select { + case <-ctx.Done(): + return + default: + } + + // KV entry not updated - do nothing + if meta != nil && meta.LastIndex == lastIndex { + logger.Trace("KV value unchanged") + continue + } + + // update the data within our cache + cs.mu.Lock() + val, ok := cs.cache[kvReq.cacheKey()] + if !ok { + logger.Error("KV data is not present within the cache - stopping blocking query refresher") + cs.mu.Unlock() + return + } + val.kvPair = pair + val.err = err + cs.mu.Unlock() + + // now determine the next wait index + if err == nil { + lastIndex = meta.LastIndex + failures = 0 + } else { + // reset the last index to 0 to do a non-blocking query + lastIndex = 0 + // Set up for exponental backoff + failures++ + + retry := retryInterval * time.Duration(failures*failures) + if retry > maxBackoffTime { + retry = maxBackoffTime + } + logger.Error("KV Get errored", "error", err, "retry", retry.String()) + select { + case <-time.After(retry): + case <-ctx.Done(): + return + } + } + } + }() +} + +func (cs *ConfigSourcer) stop() error { + cs.mu.Lock() + defer cs.mu.Unlock() + + // Stop all our background renewers + for _, v := range cs.cache { + if v.cancel != nil { + v.cancel() + } + } + + cs.cache = nil + + return nil +} + +func (cs *ConfigSourcer) Documentation() (*docs.Documentation, error) { + doc, err := docs.New(docs.RequestFromStruct(&reqConfig{})) + if err != nil { + return nil, err + } + + doc.Description("Read configuration values from the Consul KV store.") + + doc.SetRequestField( + "key", + "the KV path to retrieve", + ) + + doc.SetRequestField( + "namespace", + "the namespace to load the KV value from.", + docs.Summary( + "If not specified then it will default to the plugin's global namespace", + "configuration. If that is also not specified then Consul will default", + "the namespace like it would any other request.", + ), + ) + + doc.SetRequestField( + "partition", + "the partition to load the KV value from.", + docs.Summary( + "If not specified then it will default to the plugin's global partition", + "configuration. If that is also not specified then Consul will default", + "the partition like it would any other request.", + ), + ) + + doc.SetRequestField( + "datacenter", + "the datacenter to load the KV value from.", + docs.Summary( + "If not specified then it will default to the plugin's global datacenter", + "configuration. If that is also not specified then Consul will default", + "the datacenter like it would any other request.", + ), + ) + + doc.SetRequestField( + "allow_stale", + "whether to perform a stale query for retrieving the KV data", + docs.Summary( + "If not set this will default to true. It must explicitly be set to false", + "in order to use consistent queries.", + ), + ) + + return doc, nil +} diff --git a/builtin/consul/consul.go b/builtin/consul/consul.go new file mode 100644 index 00000000000..3f91a08d8a7 --- /dev/null +++ b/builtin/consul/consul.go @@ -0,0 +1,11 @@ +// Package consul contains components for syncing app configuration with Consul. +package consul + +import ( + "github.com/hashicorp/waypoint-plugin-sdk" +) + +// Options are the SDK options to use for instantiation for this plugin. +var Options = []sdk.Option{ + sdk.WithComponents(&ConfigSourcer{}), +} diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index b5eb58a66b9..a53f9a9d85e 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -17,6 +17,7 @@ import ( lambdaFunctionUrl "github.com/hashicorp/waypoint/builtin/aws/lambda/function_url" "github.com/hashicorp/waypoint/builtin/aws/ssm" "github.com/hashicorp/waypoint/builtin/azure/aci" + "github.com/hashicorp/waypoint/builtin/consul" "github.com/hashicorp/waypoint/builtin/docker" dockerpull "github.com/hashicorp/waypoint/builtin/docker/pull" "github.com/hashicorp/waypoint/builtin/exec" @@ -63,6 +64,7 @@ var ( "vault": vault.Options, "terraform-cloud": tfc.Options, "null": null.Options, + "consul": consul.Options, } // BaseFactories is the set of base plugin factories. This will include any @@ -97,6 +99,9 @@ var ( "terraform-cloud": { Component: &tfc.ConfigSourcer{}, }, + "consul": { + Component: &consul.ConfigSourcer{}, + }, } ) From e28bf0e21e29e9ff639003bfeda6e0e9fd8a149b Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:08:16 -0400 Subject: [PATCH 02/10] Add Consul deps. --- go.mod | 8 ++++---- go.sum | 27 ++++++++++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 9e8e1002ceb..033e610d513 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 github.com/hashicorp/aws-sdk-go-base v0.7.0 github.com/hashicorp/cap v0.1.1 + github.com/hashicorp/consul/api v1.15.2 github.com/hashicorp/go-argmapper v0.2.4 github.com/hashicorp/go-bexpr v0.1.10 github.com/hashicorp/go-cleanhttp v0.5.2 @@ -147,7 +148,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect github.com/apex/log v1.9.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect - github.com/armon/go-metrics v0.3.9 // indirect + github.com/armon/go-metrics v0.3.10 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -208,7 +209,6 @@ require ( github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect - github.com/hashicorp/consul/api v1.7.0 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect @@ -222,7 +222,7 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/protostructure v0.0.0-20220321173139-813f7b927cb7 // indirect - github.com/hashicorp/serf v0.9.3 // indirect + github.com/hashicorp/serf v0.9.7 // indirect github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864 // indirect github.com/heroku/color v0.0.6 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -250,7 +250,7 @@ require ( github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/miekg/dns v1.1.27 // indirect + github.com/miekg/dns v1.1.41 // indirect github.com/mitchellh/go-server-timing v1.0.0 // indirect github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/go.sum b/go.sum index 95f90148e3f..55428d6a865 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,9 @@ github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -1024,11 +1025,13 @@ github.com/hashicorp/aws-sdk-go-base v0.7.0/go.mod h1:2fRjWDv3jJBeN6mVWFHV6hFTNe github.com/hashicorp/cap v0.1.1 h1:GjO4+9+H0wv/89YoEsxeVc2jIizL19r5v5l2lpaH8Kg= github.com/hashicorp/cap v0.1.1/go.mod h1:VfBvK2ULRyqsuqAnjgZl7HJ7/CGMC7ro4H5eXiZuun8= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.7.0 h1:tGs8Oep67r8CcA2Ycmb/8BLBcJ70St44mF2X10a/qPg= github.com/hashicorp/consul/api v1.7.0/go.mod h1:1NSuaUUkFaJzMasbfq/11wKYWSR67Xn6r2DXKhuDNFg= +github.com/hashicorp/consul/api v1.15.2 h1:3Q/pDqvJ7udgt/60QOOW/p/PeKioQN+ncYzzCdN2av0= +github.com/hashicorp/consul/api v1.15.2/go.mod h1:v6nvB10borjOuIwNRZYPZiHKrTM/AyrGtd0WVVodKM8= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw= github.com/hashicorp/consul/sdk v0.6.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= +github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4= +github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c= github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -1068,8 +1071,9 @@ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-kms-wrapping/entropy/v2 v2.0.0/go.mod h1:xvb32K2keAc+R8DSFG2IwDcydK9DBQE+fGA5fsw6hSk= github.com/hashicorp/go-memdb v1.3.2 h1:RBKHOsnSszpU6vxq80LzC2BaQjuuvoyaQbkLTf7V7g8= github.com/hashicorp/go-memdb v1.3.2/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= @@ -1131,9 +1135,12 @@ github.com/hashicorp/horizon v0.0.0-20210317214650-d2053943be04/go.mod h1:Tvirjj github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= +github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/nomad/api v0.0.0-20220510192829-894c2e61dd03 h1:iMYuZcrU+zblX3Zu90jmTHfvnIfHqqFEzJcsDlFNEO4= github.com/hashicorp/nomad/api v0.0.0-20220510192829-894c2e61dd03/go.mod h1:b/AoT79m3PEpb6tKCFKva/M+q1rKJNUk5mdu1S8DymM= github.com/hashicorp/opaqueany v0.0.0-20220321170339-a5c6ff5bb0ec h1:WfdoyL0vJ+mQWaUdzNMkk+o1ACVa6aO2i9AzGPboF5k= @@ -1141,8 +1148,9 @@ github.com/hashicorp/opaqueany v0.0.0-20220321170339-a5c6ff5bb0ec/go.mod h1:adXe github.com/hashicorp/protostructure v0.0.0-20220321173139-813f7b927cb7 h1:jTrmnIPP65IvMLaFr05QzwGGHpK2rMpwFxqh9n3nv3o= github.com/hashicorp/protostructure v0.0.0-20220321173139-813f7b927cb7/go.mod h1:FE2g4U8UsIjarjyL4FgNUqYw0XMtsUL5kN7qpckyQv4= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.3 h1:AVF6JDQQens6nMHT9OGERBvK0f8rPrAGILnsKLr6lzM= github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= +github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/uuid v0.0.0-20160311170451-ebb0a03e909c/go.mod h1:fHzc09UnyJyqyW+bFuq864eh+wC7dj65aXmXLRe5to0= github.com/hashicorp/vault/api v1.0.5-0.20190909201928-35325e2c3262/go.mod h1:LGTA4eiQKhPGTBgi6fCuAT5n0S3CJBHa7cpUotrLxjw= github.com/hashicorp/vault/api v1.8.0 h1:7765sW1XBt+qf4XKIYE4ebY9qc/yi9V2/egzGSUNMZU= @@ -1389,8 +1397,9 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -2143,9 +2152,11 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2284,6 +2295,7 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2299,6 +2311,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 2acfecded3bff850cf4146aa862c1cb3060505ef Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:19:22 -0400 Subject: [PATCH 03/10] Add Consul config sourcer docs. --- .../partials/components/builder-consul.mdx | 11 ++++ .../components/configsourcer-consul.mdx | 53 +++++++++++++++++++ .../partials/components/platform-consul.mdx | 11 ++++ .../partials/components/registry-consul.mdx | 11 ++++ .../components/releasemanager-consul.mdx | 11 ++++ .../partials/components/task-consul.mdx | 11 ++++ website/content/plugins/consul.mdx | 9 ++++ website/data/plugins-nav-data.json | 4 ++ 8 files changed, 121 insertions(+) create mode 100644 website/content/partials/components/builder-consul.mdx create mode 100644 website/content/partials/components/configsourcer-consul.mdx create mode 100644 website/content/partials/components/platform-consul.mdx create mode 100644 website/content/partials/components/registry-consul.mdx create mode 100644 website/content/partials/components/releasemanager-consul.mdx create mode 100644 website/content/partials/components/task-consul.mdx create mode 100644 website/content/plugins/consul.mdx diff --git a/website/content/partials/components/builder-consul.mdx b/website/content/partials/components/builder-consul.mdx new file mode 100644 index 00000000000..0f9b7aec3a4 --- /dev/null +++ b/website/content/partials/components/builder-consul.mdx @@ -0,0 +1,11 @@ +## consul (builder) + +### Interface + +### Required Parameters + +This plugin has no required parameters. + +### Optional Parameters + +This plugin has no optional parameters. diff --git a/website/content/partials/components/configsourcer-consul.mdx b/website/content/partials/components/configsourcer-consul.mdx new file mode 100644 index 00000000000..bb3b0ce1745 --- /dev/null +++ b/website/content/partials/components/configsourcer-consul.mdx @@ -0,0 +1,53 @@ +## consul (configsourcer) + +Read configuration values from the Consul KV store. + +### Required Parameters + +These parameters are used in `dynamic` for sourcing [configuration values](/docs/app-config/dynamic) or [input variable values](/docs/waypoint-hcl/variables/dynamic). + +#### key + +The KV path to retrieve. + +- Type: **string** + +### Optional Parameters + +These parameters are used in `dynamic` for sourcing [configuration values](/docs/app-config/dynamic) or [input variable values](/docs/waypoint-hcl/variables/dynamic). + +#### allow_stale + +Whether to perform a stale query for retrieving the KV data. + +If not set this will default to true. It must explicitly be set to false in order to use consistent queries. + +- Type: **bool** +- **Optional** + +#### datacenter + +The datacenter to load the KV value from. + +If not specified then it will default to the plugin's global datacenter configuration. If that is also not specified then Consul will default the datacenter like it would any other request. + +- Type: **string** +- **Optional** + +#### namespace + +The namespace to load the KV value from. + +If not specified then it will default to the plugin's global namespace configuration. If that is also not specified then Consul will default the namespace like it would any other request. + +- Type: **string** +- **Optional** + +#### partition + +The partition to load the KV value from. + +If not specified then it will default to the plugin's global partition configuration. If that is also not specified then Consul will default the partition like it would any other request. + +- Type: **string** +- **Optional** diff --git a/website/content/partials/components/platform-consul.mdx b/website/content/partials/components/platform-consul.mdx new file mode 100644 index 00000000000..522529ca137 --- /dev/null +++ b/website/content/partials/components/platform-consul.mdx @@ -0,0 +1,11 @@ +## consul (platform) + +### Interface + +### Required Parameters + +This plugin has no required parameters. + +### Optional Parameters + +This plugin has no optional parameters. diff --git a/website/content/partials/components/registry-consul.mdx b/website/content/partials/components/registry-consul.mdx new file mode 100644 index 00000000000..fae52507a83 --- /dev/null +++ b/website/content/partials/components/registry-consul.mdx @@ -0,0 +1,11 @@ +## consul (registry) + +### Interface + +### Required Parameters + +This plugin has no required parameters. + +### Optional Parameters + +This plugin has no optional parameters. diff --git a/website/content/partials/components/releasemanager-consul.mdx b/website/content/partials/components/releasemanager-consul.mdx new file mode 100644 index 00000000000..b68bb1d7852 --- /dev/null +++ b/website/content/partials/components/releasemanager-consul.mdx @@ -0,0 +1,11 @@ +## consul (releasemanager) + +### Interface + +### Required Parameters + +This plugin has no required parameters. + +### Optional Parameters + +This plugin has no optional parameters. diff --git a/website/content/partials/components/task-consul.mdx b/website/content/partials/components/task-consul.mdx new file mode 100644 index 00000000000..35ea43327ae --- /dev/null +++ b/website/content/partials/components/task-consul.mdx @@ -0,0 +1,11 @@ +## consul (task) + +### Interface + +### Required Parameters + +This plugin has no required parameters. + +### Optional Parameters + +This plugin has no optional parameters. diff --git a/website/content/plugins/consul.mdx b/website/content/plugins/consul.mdx new file mode 100644 index 00000000000..8886e1948e9 --- /dev/null +++ b/website/content/plugins/consul.mdx @@ -0,0 +1,9 @@ +--- +layout: plugins +page_title: 'Plugin: Consul' +description: 'Source configuration from Consul KV' +--- + +# Consul + +@include "components/configsourcer-consul.mdx" diff --git a/website/data/plugins-nav-data.json b/website/data/plugins-nav-data.json index 4f130ad1ebf..bf36283c64e 100644 --- a/website/data/plugins-nav-data.json +++ b/website/data/plugins-nav-data.json @@ -19,6 +19,10 @@ "title": "azure-container-instance", "path": "azure-container-instance" }, + { + "title": "consul", + "path": "consul" + }, { "title": "docker", "path": "docker" From 8e0fb4248b8636e0c958b5bba4ce7f0add11bf03 Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:21:01 -0400 Subject: [PATCH 04/10] Changelog. --- .changelog/4045.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/4045.txt diff --git a/.changelog/4045.txt b/.changelog/4045.txt new file mode 100644 index 00000000000..5bbe4435602 --- /dev/null +++ b/.changelog/4045.txt @@ -0,0 +1,3 @@ +```release-note:feature +plugin/consul: Consul key-value data config sourcer plugin +``` \ No newline at end of file From 5b671f4ee5809a42f2a3a68ed0aafeeccacbe8db Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Thu, 13 Oct 2022 09:56:09 -0400 Subject: [PATCH 05/10] maint: small error updates, function and type renaming. --- builtin/consul/config_sourcer.go | 44 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index 314b9145941..360382c9ba8 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -43,14 +43,16 @@ type cachedVal struct { err error } +// ConfigSourcerConfig is used to configure where to talk to Consul, and from +// where the KV data is to be retrieved type ConfigSourcerConfig struct { // Configuration for where to talk to Consul Address string `hcl:"address,optional"` Scheme string `hcl:"scheme,optional"` - HTTPAuth ConsulHTTPAuth `hcl:"http_auth,optional"` + HTTPAuth consulHTTPAuth `hcl:"http_auth,optional"` Token string `hcl:"token,optional"` TokenFile string `hcl:"token_file,optional"` - TLSConfig TLSConfig `hcl:"tls,optional"` + TLSConfig tlsConfig `hcl:"tls,optional"` // Default location of KV data Datacenter string `hcl:"datacenter,optional"` @@ -58,7 +60,7 @@ type ConfigSourcerConfig struct { Partition string `hcl:"partition,optional"` } -func (conf *ConfigSourcerConfig) toAPIClient() (*api.Client, error) { +func (conf *ConfigSourcerConfig) client() (*api.Client, error) { apiConfig := api.Config{ Address: conf.Address, Scheme: conf.Scheme, @@ -74,7 +76,7 @@ func (conf *ConfigSourcerConfig) toAPIClient() (*api.Client, error) { return api.NewClient(&apiConfig) } -type TLSConfig struct { +type tlsConfig struct { ServerName string `hcl:"server_name,optional"` CAFile string `hcl:"ca_file,optional"` CAPath string `hcl:"ca_path,optional"` @@ -86,7 +88,7 @@ type TLSConfig struct { InsecureHTTPs bool `hcl:"insecure_https,optional"` } -func (t *TLSConfig) toAPITLSConfig() *api.TLSConfig { +func (t *tlsConfig) toAPITLSConfig() *api.TLSConfig { return &api.TLSConfig{ Address: t.ServerName, CAFile: t.CAFile, @@ -100,12 +102,12 @@ func (t *TLSConfig) toAPITLSConfig() *api.TLSConfig { } } -type ConsulHTTPAuth struct { +type consulHTTPAuth struct { Username string `hcl:"username,optional"` Password string `hcl:"password,optional"` } -func (a *ConsulHTTPAuth) toApiAuth() *api.HttpBasicAuth { +func (a *consulHTTPAuth) toApiAuth() *api.HttpBasicAuth { if a.Username == "" && a.Password == "" { return nil } @@ -124,23 +126,23 @@ type ConfigSourcer struct { cache map[string]*cachedVal } -// Implement Configurable +// Config implements the Configurable interface func (cs *ConfigSourcer) Config() (interface{}, error) { return &cs.config, nil } -// Implement ConfigurableNotify +// ConfigSet implements the ConfigurableNotify interface func (cs *ConfigSourcer) ConfigSet(config interface{}) error { conf, ok := config.(*ConfigSourcerConfig) if !ok { // The Waypoint SDK should ensure this never gets hit - return fmt.Errorf("Expected *ConfigSourcerConfig as parameter") + return fmt.Errorf("expected *ConfigSourcerConfig as parameter") } // attempt to create API client - client, err := conf.toAPIClient() + client, err := conf.client() if err != nil { - return fmt.Errorf("Invalid configuration: %w", err) + return fmt.Errorf("invalid configuration: %w", err) } cs.client = client.KV() @@ -178,7 +180,7 @@ func (cs *ConfigSourcer) StopFunc() interface{} { func (cs *ConfigSourcer) read(ctx context.Context, log hclog.Logger, reqs []*component.ConfigRequest) ([]*pb.ConfigSource_Value, error) { log.Trace("Reading KV data from Consul") - // Setup our lock + // Set up our lock cs.mu.Lock() defer cs.mu.Unlock() @@ -189,9 +191,9 @@ func (cs *ConfigSourcer) read(ctx context.Context, log hclog.Logger, reqs []*com // Create our Consul API client if this is our first time if cs.client == nil { - client, err := cs.config.toAPIClient() + client, err := cs.config.client() if err != nil { - return nil, fmt.Errorf("Invalid Consul client configuration: %w", err) + return nil, fmt.Errorf("invalid Consul client configuration: %w", err) } cs.client = client.KV() } @@ -283,11 +285,6 @@ func (cs *ConfigSourcer) startBlockingQuery(ctx context.Context, logger hclog.Lo // namespaces and partitions except with the global client defaults and thus isn't suitable for this usage. for { - logger.Trace("Issuing blocking query", "wait-index", lastIndex) - // set the wait index to use for the query - opts.WaitIndex = lastIndex - pair, meta, err := cs.client.Get(kvReq.Key, opts) - // check if we are being stopped select { case <-ctx.Done(): @@ -295,6 +292,11 @@ func (cs *ConfigSourcer) startBlockingQuery(ctx context.Context, logger hclog.Lo default: } + logger.Trace("Issuing blocking query", "wait-index", lastIndex) + // set the wait index to use for the query + opts.WaitIndex = lastIndex + pair, meta, err := cs.client.Get(kvReq.Key, opts) + // KV entry not updated - do nothing if meta != nil && meta.LastIndex == lastIndex { logger.Trace("KV value unchanged") @@ -320,7 +322,7 @@ func (cs *ConfigSourcer) startBlockingQuery(ctx context.Context, logger hclog.Lo } else { // reset the last index to 0 to do a non-blocking query lastIndex = 0 - // Set up for exponental backoff + // Set up for exponential backoff failures++ retry := retryInterval * time.Duration(failures*failures) From efeae6c754dcede1ebc8bf4a85c016d82b774d74 Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Thu, 13 Oct 2022 12:15:34 -0400 Subject: [PATCH 06/10] refactor: start Consul blocking query in goroutine. The function to start a Consul blocking query will now be called via a goroutine, rather than the contents of the function being entirely wrapped inside an anonymous function, called via a goroutine. --- builtin/consul/config_sourcer.go | 112 ++++++++++++++++--------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index 360382c9ba8..d0930863dc0 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -252,7 +252,7 @@ func (cs *ConfigSourcer) read(ctx context.Context, log hclog.Logger, reqs []*com err: nil, } cs.cache[kvReq.cacheKey()] = cacheVal - cs.startBlockingQuery(refreshCtx, log, kvReq, meta.LastIndex, opts) + go cs.startConsulBlockingQuery(refreshCtx, log, kvReq, meta.LastIndex, opts) } if cacheVal.err != nil { @@ -277,67 +277,69 @@ func (cs *ConfigSourcer) read(ctx context.Context, log hclog.Logger, reqs []*com return results, nil } -func (cs *ConfigSourcer) startBlockingQuery(ctx context.Context, logger hclog.Logger, kvReq reqConfig, lastIndex uint64, opts *api.QueryOptions) { - go func() { - opts := opts.WithContext(ctx) - failures := 0 - // Ideally we would use the github.com/hashicorp/consul/api/watch package. However that package doesn't support - // namespaces and partitions except with the global client defaults and thus isn't suitable for this usage. +// startConsulBlockingQuery starts a blocking query to the Consul API to poll +// for changes in the KV data store at the specified path. This is expected to +// be called via a goroutine. For more information on Consul blocking queries, see: +// https://developer.hashicorp.com/consul/api-docs/features/blocking +func (cs *ConfigSourcer) startConsulBlockingQuery(ctx context.Context, logger hclog.Logger, kvReq reqConfig, lastIndex uint64, opts *api.QueryOptions) { + opts = opts.WithContext(ctx) + failures := 0 + // Ideally we would use the github.com/hashicorp/consul/api/watch package. However that package doesn't support + // namespaces and partitions except with the global client defaults and thus isn't suitable for this usage. + + for { + // check if we are being stopped + select { + case <-ctx.Done(): + return + default: + } - for { - // check if we are being stopped - select { - case <-ctx.Done(): - return - default: - } + logger.Trace("Issuing blocking query", "wait-index", lastIndex) + // set the wait index to use for the query + opts.WaitIndex = lastIndex + pair, meta, err := cs.client.Get(kvReq.Key, opts) - logger.Trace("Issuing blocking query", "wait-index", lastIndex) - // set the wait index to use for the query - opts.WaitIndex = lastIndex - pair, meta, err := cs.client.Get(kvReq.Key, opts) + // KV entry not updated - do nothing + if meta != nil && meta.LastIndex == lastIndex { + logger.Trace("KV value unchanged") + continue + } - // KV entry not updated - do nothing - if meta != nil && meta.LastIndex == lastIndex { - logger.Trace("KV value unchanged") - continue + // update the data within our cache + cs.mu.Lock() + val, ok := cs.cache[kvReq.cacheKey()] + if !ok { + logger.Error("KV data is not present within the cache - stopping blocking query refresher") + cs.mu.Unlock() + return + } + val.kvPair = pair + val.err = err + cs.mu.Unlock() + + // now determine the next wait index + if err == nil { + lastIndex = meta.LastIndex + failures = 0 + } else { + // reset the last index to 0 to do a non-blocking query + lastIndex = 0 + // Set up for exponential backoff + failures++ + + retry := retryInterval * time.Duration(failures*failures) + if retry > maxBackoffTime { + retry = maxBackoffTime } - - // update the data within our cache - cs.mu.Lock() - val, ok := cs.cache[kvReq.cacheKey()] - if !ok { - logger.Error("KV data is not present within the cache - stopping blocking query refresher") - cs.mu.Unlock() + logger.Error("KV Get errored", "error", err, "retry", retry.String()) + select { + case <-time.After(retry): + case <-ctx.Done(): return } - val.kvPair = pair - val.err = err - cs.mu.Unlock() - - // now determine the next wait index - if err == nil { - lastIndex = meta.LastIndex - failures = 0 - } else { - // reset the last index to 0 to do a non-blocking query - lastIndex = 0 - // Set up for exponential backoff - failures++ - - retry := retryInterval * time.Duration(failures*failures) - if retry > maxBackoffTime { - retry = maxBackoffTime - } - logger.Error("KV Get errored", "error", err, "retry", retry.String()) - select { - case <-time.After(retry): - case <-ctx.Done(): - return - } - } } - }() + } } func (cs *ConfigSourcer) stop() error { From 56ca26b82bec38f5264fe894d4c5d52df4b2b799 Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Thu, 13 Oct 2022 13:08:13 -0400 Subject: [PATCH 07/10] chore: simplify error handling for checking cache in Consul plugin. --- builtin/consul/config_sourcer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index d0930863dc0..c4478e400a2 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -319,10 +319,7 @@ func (cs *ConfigSourcer) startConsulBlockingQuery(ctx context.Context, logger hc cs.mu.Unlock() // now determine the next wait index - if err == nil { - lastIndex = meta.LastIndex - failures = 0 - } else { + if err != nil { // reset the last index to 0 to do a non-blocking query lastIndex = 0 // Set up for exponential backoff @@ -339,6 +336,8 @@ func (cs *ConfigSourcer) startConsulBlockingQuery(ctx context.Context, logger hc return } } + lastIndex = meta.LastIndex + failures = 0 } } From ff3a948c6a1ce93a5a4a421a71b629cc41ed0b16 Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:26:52 -0400 Subject: [PATCH 08/10] chore: remove single-use helper function for TLS config formatting. --- builtin/consul/config_sourcer.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index c4478e400a2..0436821974c 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -70,7 +70,17 @@ func (conf *ConfigSourcerConfig) client() (*api.Client, error) { TokenFile: conf.TokenFile, Namespace: conf.Namespace, Partition: conf.Partition, - TLSConfig: *conf.TLSConfig.toAPITLSConfig(), + TLSConfig: api.TLSConfig{ + Address: conf.TLSConfig.ServerName, + CAFile: conf.TLSConfig.CAFile, + CAPath: conf.TLSConfig.CAPath, + CAPem: conf.TLSConfig.CAPem, + CertFile: conf.TLSConfig.CertFile, + CertPEM: conf.TLSConfig.CertPEM, + KeyFile: conf.TLSConfig.KeyFile, + KeyPEM: conf.TLSConfig.KeyPEM, + InsecureSkipVerify: conf.TLSConfig.InsecureHTTPs, + }, } return api.NewClient(&apiConfig) @@ -88,20 +98,6 @@ type tlsConfig struct { InsecureHTTPs bool `hcl:"insecure_https,optional"` } -func (t *tlsConfig) toAPITLSConfig() *api.TLSConfig { - return &api.TLSConfig{ - Address: t.ServerName, - CAFile: t.CAFile, - CAPath: t.CAPath, - CAPem: t.CAPem, - CertFile: t.CertFile, - CertPEM: t.CertPEM, - KeyFile: t.KeyFile, - KeyPEM: t.KeyPEM, - InsecureSkipVerify: t.InsecureHTTPs, - } -} - type consulHTTPAuth struct { Username string `hcl:"username,optional"` Password string `hcl:"password,optional"` From c7379271d8f9e398a5e861a22dde432fddb8d389 Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:32:03 -0400 Subject: [PATCH 09/10] chore: return error with status code for Consul plugin config check. --- builtin/consul/config_sourcer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index 0436821974c..548d85336ee 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -132,7 +132,7 @@ func (cs *ConfigSourcer) ConfigSet(config interface{}) error { conf, ok := config.(*ConfigSourcerConfig) if !ok { // The Waypoint SDK should ensure this never gets hit - return fmt.Errorf("expected *ConfigSourcerConfig as parameter") + return status.Errorf(codes.FailedPrecondition, "expected *ConfigSourcerConfig as parameter") } // attempt to create API client From 22a277f59dd452fc5946c331babb08bcaea4f2cd Mon Sep 17 00:00:00 2001 From: Joseph Rajewski <83741749+paladin-devops@users.noreply.github.com> Date: Wed, 19 Oct 2022 16:11:05 -0400 Subject: [PATCH 10/10] maint: avoid slamming Consul API. Update Consul config sourcer plugin to sleep for 5 seconds before hitting the Consul KV API again, if there is no value change. --- builtin/consul/config_sourcer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builtin/consul/config_sourcer.go b/builtin/consul/config_sourcer.go index 548d85336ee..86de908ca16 100644 --- a/builtin/consul/config_sourcer.go +++ b/builtin/consul/config_sourcer.go @@ -299,6 +299,7 @@ func (cs *ConfigSourcer) startConsulBlockingQuery(ctx context.Context, logger hc // KV entry not updated - do nothing if meta != nil && meta.LastIndex == lastIndex { logger.Trace("KV value unchanged") + time.Sleep(retryInterval) continue }