From 654fc4bfda28943f2062a0f48a5c5aa7811fd9ce Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Apr 2020 13:50:35 +0300 Subject: [PATCH 01/18] Move filekeystore to Mapper Signed-off-by: chrismark --- filebeat/autodiscover/builder/hints/logs.go | 4 +-- .../autodiscover/builder/hints/monitors.go | 4 +-- .../autodiscover/appenders/config/config.go | 2 +- .../autodiscover/providers/docker/docker.go | 8 ++--- .../autodiscover/providers/jolokia/jolokia.go | 8 ++--- .../providers/kubernetes/kubernetes.go | 6 +--- .../providers/kubernetes/node_test.go | 10 ++---- .../providers/kubernetes/pod_test.go | 11 ++---- .../providers/kubernetes/service_test.go | 10 ++---- libbeat/autodiscover/template/config.go | 36 ++++++++----------- libbeat/autodiscover/template/config_test.go | 9 +++-- .../autodiscover/builder/hints/metrics.go | 4 +-- .../providers/aws/ec2/provider.go | 4 +-- .../providers/aws/elb/provider.go | 4 +-- 14 files changed, 39 insertions(+), 81 deletions(-) diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index e2f37caee74e..457a948c93c7 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -109,7 +109,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config { } logp.Debug("hints.builder", "generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, false) + return template.ApplyConfigTemplate(event, configs, nil) } tempCfg := common.MapStr{} @@ -163,7 +163,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config { logp.Debug("hints.builder", "generated config %+v", config) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, []*common.Config{config}, false) + return template.ApplyConfigTemplate(event, []*common.Config{config}, nil) } func (l *logHints) getMultiline(hints common.MapStr) common.MapStr { diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 836b5a9326c5..431dbe916344 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -91,7 +91,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config { } hb.logger.Debugf("generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, false) + return template.ApplyConfigTemplate(event, configs, nil) } tempCfg := common.MapStr{} @@ -121,7 +121,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config { } // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, false) + return template.ApplyConfigTemplate(event, configs, nil) } func (hb *heartbeatHints) getType(hints common.MapStr) common.MapStr { diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 60f8a543f4a0..156fcdf07b26 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -104,7 +104,7 @@ func (c *configAppender) Append(event bus.Event) { } // Apply the template - template.ApplyConfigTemplate(event, cfgs, false) + template.ApplyConfigTemplate(event, cfgs, nil) } // Replace old config with newly appended configs diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 9bfa13000b1b..f2600195b8f7 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -56,7 +56,6 @@ type Provider struct { stoppers map[string]*time.Timer stopTrigger chan *dockerContainerMetadata logger *logp.Logger - keystore keystore.Keystore } // AutodiscoverBuilder builds and returns an autodiscover provider @@ -78,11 +77,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(err) } - mapper, err := template.NewConfigMapper(config.Templates) + mapper, err := template.NewConfigMapper(config.Templates, keystore) if err != nil { return nil, errWrap(err) } - if len(mapper) == 0 && !config.Hints.Enabled() { + if len(mapper.ConditionMaps) == 0 && !config.Hints.Enabled() { return nil, errWrap(fmt.Errorf("no configs or hints defined for autodiscover provider")) } @@ -117,7 +116,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore stoppers: make(map[string]*time.Timer), stopTrigger: make(chan *dockerContainerMetadata), logger: logger, - keystore: keystore, }, nil } @@ -306,8 +304,6 @@ func (d *Provider) emitContainer(container *docker.Container, meta *dockerMetada } func (d *Provider) publish(event bus.Event) { - // attach keystore to the event to be consumed by the static configs - event["keystore"] = d.keystore // Try to match a config if config := d.templates.GetConfig(event); config != nil { event["config"] = config diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index 4a18ffffec9d..9fcb64823bf8 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -49,7 +49,6 @@ type Provider struct { appenders autodiscover.Appenders templates template.Mapper discovery DiscoveryProber - keystore keystore.Keystore } // AutodiscoverBuilder builds a Jolokia Discovery autodiscover provider, it fails if @@ -70,11 +69,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore Interfaces: config.Interfaces, } - mapper, err := template.NewConfigMapper(config.Templates) + mapper, err := template.NewConfigMapper(config.Templates, keystore) if err != nil { return nil, errWrap(err) } - if len(mapper) == 0 { + if len(mapper.ConditionMaps) == 0 { return nil, errWrap(fmt.Errorf("no configs defined for autodiscover provider")) } @@ -94,7 +93,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore builders: builders, appenders: appenders, discovery: discovery, - keystore: keystore, }, nil } @@ -109,8 +107,6 @@ func (p *Provider) Start() { } func (p *Provider) publish(event bus.Event) { - // attach keystore to the event to be consumed by the static configs - event["keystore"] = p.keystore if config := p.templates.GetConfig(event); config != nil { event["config"] = config } else if config := p.builders.GetConfig(event); config != nil { diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 4a4a4566f8e6..2283cec77478 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -55,7 +55,6 @@ type Provider struct { appenders autodiscover.Appenders logger *logp.Logger eventer Eventer - keystore keystore.Keystore } // AutodiscoverBuilder builds and returns an autodiscover provider @@ -77,7 +76,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(err) } - mapper, err := template.NewConfigMapper(config.Templates) + mapper, err := template.NewConfigMapper(config.Templates, keystore) if err != nil { return nil, errWrap(err) } @@ -99,7 +98,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore builders: builders, appenders: appenders, logger: logger, - keystore: keystore, } switch config.Resource { @@ -138,8 +136,6 @@ func (p *Provider) String() string { } func (p *Provider) publish(event bus.Event) { - // attach keystore to the event to be consumed by the static configs - event["keystore"] = p.keystore // Try to match a config if config := p.templates.GetConfig(event); config != nil { event["config"] = config diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index f2fbe78dba6e..9180f3291fa6 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -21,8 +21,6 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" - "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -33,7 +31,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -113,7 +111,6 @@ func TestGenerateHints_Node(t *testing.T) { } func TestEmitEvent_Node(t *testing.T) { - k, _ := keystore.NewFileKeystore("test") name := "metricbeat" nodeIP := "192.168.0.1" uid := "005f3b90-4b9d-12f8-acf0-31020a840133" @@ -162,7 +159,6 @@ func TestEmitEvent_Node(t *testing.T) { "host": "192.168.0.1", "id": uid, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "node": common.MapStr{ "name": "metricbeat", @@ -222,7 +218,6 @@ func TestEmitEvent_Node(t *testing.T) { "host": "", "id": uid, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "node": common.MapStr{ "name": "metricbeat", @@ -245,7 +240,7 @@ func TestEmitEvent_Node(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil) + mapper, err := template.NewConfigMapper(nil, nil) if err != nil { t.Fatal(err) } @@ -256,7 +251,6 @@ func TestEmitEvent_Node(t *testing.T) { bus: bus.New(logp.NewLogger("bus"), "test"), templates: mapper, logger: logp.NewLogger("kubernetes"), - keystore: k, } no := &node{ diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 05b50987b2e8..2b14dbb3d42c 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -21,8 +21,6 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" - "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -33,7 +31,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -333,7 +331,6 @@ func TestGenerateHints(t *testing.T) { } func TestEmitEvent(t *testing.T) { - k, _ := keystore.NewFileKeystore("test") name := "filebeat" namespace := "default" podIP := "127.0.0.1" @@ -397,7 +394,6 @@ func TestEmitEvent(t *testing.T) { "host": "127.0.0.1", "id": cid, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "container": common.MapStr{ "id": "foobar", @@ -530,7 +526,6 @@ func TestEmitEvent(t *testing.T) { "host": "", "id": cid, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "container": common.MapStr{ "id": "", @@ -600,7 +595,6 @@ func TestEmitEvent(t *testing.T) { "host": "127.0.0.1", "id": cid, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "container": common.MapStr{ "id": "", @@ -639,7 +633,7 @@ func TestEmitEvent(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil) + mapper, err := template.NewConfigMapper(nil, nil) if err != nil { t.Fatal(err) } @@ -650,7 +644,6 @@ func TestEmitEvent(t *testing.T) { bus: bus.New(logp.NewLogger("bus"), "test"), templates: mapper, logger: logp.NewLogger("kubernetes"), - keystore: k, } pod := &pod{ diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index 0e3c8ddb0a83..241f3f2a986a 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -21,8 +21,6 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" - "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -33,7 +31,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -234,7 +232,6 @@ func TestGenerateHints_Service(t *testing.T) { } func TestEmitEvent_Service(t *testing.T) { - k, _ := keystore.NewFileKeystore("test") name := "metricbeat" namespace := "default" clusterIP := "192.168.0.1" @@ -282,7 +279,6 @@ func TestEmitEvent_Service(t *testing.T) { "host": "192.168.0.1", "id": uid, "provider": UUID, - "keystore": k, "port": 8080, "kubernetes": common.MapStr{ "service": common.MapStr{ @@ -372,7 +368,6 @@ func TestEmitEvent_Service(t *testing.T) { "id": uid, "port": 8080, "provider": UUID, - "keystore": k, "kubernetes": common.MapStr{ "service": common.MapStr{ "name": "metricbeat", @@ -397,7 +392,7 @@ func TestEmitEvent_Service(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil) + mapper, err := template.NewConfigMapper(nil, nil) if err != nil { t.Fatal(err) } @@ -409,7 +404,6 @@ func TestEmitEvent_Service(t *testing.T) { bus: bus.New(logp.NewLogger("bus"), "test"), templates: mapper, logger: logp.NewLogger("kubernetes"), - keystore: k, } service := &service{ diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 0ce05526ecb5..e1e560e7e9c6 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -28,7 +28,10 @@ import ( // Mapper maps config templates with conditions, if a match happens on a discover event // the given template will be used as config -type Mapper []*ConditionMap +type Mapper struct { + ConditionMaps []*ConditionMap + keystore keystore.Keystore +} // ConditionMap maps a condition to the configs to use when it's triggered type ConditionMap struct { @@ -43,19 +46,19 @@ type MapperSettings []*struct { } // NewConfigMapper builds a template Mapper from given settings -func NewConfigMapper(configs MapperSettings) (mapper Mapper, err error) { +func NewConfigMapper(configs MapperSettings, keystore keystore.Keystore) (mapper Mapper, err error) { for _, c := range configs { condMap := &ConditionMap{Configs: c.Configs} if c.ConditionConfig != nil { condMap.Condition, err = conditions.NewCondition(c.ConditionConfig) if err != nil { - return nil, err + return Mapper{}, err } } - - mapper = append(mapper, condMap) + mapper.ConditionMaps = append(mapper.ConditionMaps, condMap) } + mapper.keystore = keystore return mapper, nil } @@ -74,15 +77,17 @@ func (e Event) GetValue(key string) (interface{}, error) { // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config - - for _, mapping := range c { + opts := []ucfg.Option{ + ucfg.Resolve(keystore.ResolverWrap(c.keystore)), + } + for _, mapping := range c.ConditionMaps { // An empty condition matches everything conditionOk := mapping.Condition == nil || mapping.Condition.Check(Event(event)) if mapping.Configs != nil && !conditionOk { continue } - configs := ApplyConfigTemplate(event, mapping.Configs, true) + configs := ApplyConfigTemplate(event, mapping.Configs, opts) if configs != nil { result = append(result, configs...) } @@ -91,7 +96,7 @@ func (c Mapper) GetConfig(event bus.Event) []*common.Config { } // ApplyConfigTemplate takes a set of templated configs and applys information in an event map -func ApplyConfigTemplate(event bus.Event, configs []*common.Config, keystoreEnabled bool) []*common.Config { +func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options []ucfg.Option) []*common.Config { var result []*common.Config // unpack input vars, err := ucfg.NewFrom(map[string]interface{}{ @@ -106,18 +111,7 @@ func ApplyConfigTemplate(event bus.Event, configs []*common.Config, keystoreEnab ucfg.ResolveEnv, ucfg.VarExp, } - - if keystoreEnabled { - if val, ok := event["keystore"]; ok { - eventKeystore := val.(keystore.Keystore) - opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(eventKeystore))) - delete(event, "keystore") - } - } else { - if _, ok := event["keystore"]; ok { - delete(event, "keystore") - } - } + opts = append(opts, options...) for _, config := range configs { c, err := ucfg.NewFrom(config, opts...) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index ccb27a7127af..f2bf33bf3dbb 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -87,7 +87,7 @@ func TestConfigsMapping(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings) + mapper, err := NewConfigMapper(mappings, nil) if err != nil { t.Fatal(err) } @@ -124,8 +124,7 @@ func TestConfigsMappingKeystore(t *testing.T) { - correct: config password: "${PASSWORD}"`, event: bus.Event{ - "foo": 3, - "keystore": keystore, + "foo": 3, }, expected: []*common.Config{config}, }, @@ -142,7 +141,7 @@ func TestConfigsMappingKeystore(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings) + mapper, err := NewConfigMapper(mappings, keystore) if err != nil { t.Fatal(err) } @@ -166,7 +165,7 @@ func TestNilConditionConfig(t *testing.T) { t.Fatal(err) } - _, err = NewConfigMapper(mappings) + _, err = NewConfigMapper(mappings, nil) assert.NoError(t, err) assert.Nil(t, mappings[0].ConditionConfig) } diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 1647fb9fbc75..8dc265bfce29 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -94,7 +94,7 @@ func (m *metricHints) CreateConfig(event bus.Event) []*common.Config { } logp.Debug("hints.builder", "generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, false) + return template.ApplyConfigTemplate(event, configs, nil) } @@ -155,7 +155,7 @@ func (m *metricHints) CreateConfig(event bus.Event) []*common.Config { // Apply information in event to the template to generate the final config // This especially helps in a scenario where endpoints are configured as: // co.elastic.metrics/hosts= "${data.host}:9090" - return template.ApplyConfigTemplate(event, config, false) + return template.ApplyConfigTemplate(event, config, nil) } func (m *metricHints) getModule(hints common.MapStr) string { diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 4d457c46a8cb..9374d60970ed 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -34,7 +34,6 @@ type Provider struct { stopListener bus.Listener watcher *watcher uuid uuid.UUID - keystore keystore.Keystore } // AutodiscoverBuilder is the main builder for this provider. @@ -86,7 +85,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore // internalBuilder is mainly intended for testing via mocks and stubs. // it can be configured to use a fetcher that doesn't actually hit the AWS API. func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher, keystore keystore.Keystore) (*Provider, error) { - mapper, err := template.NewConfigMapper(config.Templates) + mapper, err := template.NewConfigMapper(config.Templates, keystore) if err != nil { return nil, err } @@ -96,7 +95,6 @@ func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetche bus: bus, templates: &mapper, uuid: uuid, - keystore: keystore, } p.watcher = newWatcher( diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index 522b5ba9a4f3..f28fbf7f4cfa 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -36,7 +36,6 @@ type Provider struct { stopListener bus.Listener watcher *watcher uuid uuid.UUID - keystore keystore.Keystore } // AutodiscoverBuilder is the main builder for this provider. @@ -93,7 +92,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore // internalBuilder is mainly intended for testing via mocks and stubs. // it can be configured to use a fetcher that doesn't actually hit the AWS API. func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher, keystore keystore.Keystore) (*Provider, error) { - mapper, err := template.NewConfigMapper(config.Templates) + mapper, err := template.NewConfigMapper(config.Templates, keystore) if err != nil { return nil, err } @@ -103,7 +102,6 @@ func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetche bus: bus, templates: &mapper, uuid: uuid, - keystore: keystore, } p.watcher = newWatcher( From 33a1ff5eecd488ee41549968a9f047ffc178ea2e Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Apr 2020 16:04:58 +0300 Subject: [PATCH 02/18] Add k8s Keystore Signed-off-by: chrismark --- filebeat/autodiscover/builder/hints/logs.go | 3 +- .../autodiscover/builder/hints/logs_test.go | 4 +- libbeat/autodiscover/builder.go | 32 +++++--- libbeat/autodiscover/builder_test.go | 4 +- .../autodiscover/providers/docker/docker.go | 2 +- .../autodiscover/providers/jolokia/jolokia.go | 2 +- .../providers/kubernetes/kubernetes.go | 10 ++- libbeat/autodiscover/template/config.go | 7 +- libbeat/common/kubernetes/util.go | 10 +++ libbeat/keystore/kubernetes_keystore.go | 78 +++++++++++++++++++ .../autodiscover/builder/hints/metrics.go | 7 +- .../builder/hints/metrics_test.go | 4 +- 12 files changed, 135 insertions(+), 28 deletions(-) create mode 100644 libbeat/keystore/kubernetes_keystore.go diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 457a948c93c7..33c90a239252 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -19,6 +19,7 @@ package hints import ( "fmt" + "github.com/elastic/go-ucfg" "regexp" "github.com/elastic/beats/v7/filebeat/fileset" @@ -70,7 +71,7 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create config based on input hints in the bus event -func (l *logHints) CreateConfig(event bus.Event) []*common.Config { +func (l *logHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { var hints common.MapStr hIface, ok := event["hints"] if ok { diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index b316cdb506c2..7959431bc671 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -628,7 +628,7 @@ func TestGenerateHints(t *testing.T) { t.Fatal(err) } - cfgs := l.CreateConfig(test.event) + cfgs := l.CreateConfig(test.event, nil) assert.Equal(t, len(cfgs), test.len, test.msg) if test.len != 0 { config := common.MapStr{} @@ -860,7 +860,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { t.Fatal(err) } - cfgs := l.CreateConfig(test.event) + cfgs := l.CreateConfig(test.event, nil) assert.Equal(t, len(cfgs), test.len, test.msg) if test.len != 0 { config := common.MapStr{} diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index b77ef847a97b..cb9cbeb47140 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -20,6 +20,8 @@ package autodiscover import ( "errors" "fmt" + "github.com/elastic/beats/libbeat/keystore" + "github.com/elastic/go-ucfg" "strings" "github.com/elastic/beats/v7/libbeat/common" @@ -29,11 +31,14 @@ import ( // Builder provides an interface by which configs can be built from provider metadata type Builder interface { // CreateConfig creates a config from hints passed from providers - CreateConfig(event bus.Event) []*common.Config + CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config } -// Builders is a list of Builder objects -type Builders []Builder +// Builders is a struct of Builder list objects and a common Keystore object +type Builders struct { + Builders []Builder + keystore keystore.Keystore +} // BuilderConstructor is a func used to generate a Builder object type BuilderConstructor func(*common.Config) (Builder, error) @@ -89,9 +94,14 @@ func (r *registry) BuildBuilder(c *common.Config) (Builder, error) { // GetConfig creates configs for all builders initialized. func (b Builders) GetConfig(event bus.Event) []*common.Config { configs := []*common.Config{} - - for _, builder := range b { - if config := builder.CreateConfig(event); config != nil { + var opts []ucfg.Option + if b.keystore != nil { + opts = []ucfg.Option{ + ucfg.Resolve(keystore.ResolverWrap(b.keystore)), + } + } + for _, builder := range b.Builders { + if config := builder.CreateConfig(event, opts); config != nil { configs = append(configs, config...) } } @@ -101,11 +111,11 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { // NewBuilders instances the given list of builders. hintsCfg holds `hints` settings // for simplified mode (single 'hints' builder) -func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) { +func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, keystore keystore.Keystore) (Builders, error) { var builders Builders if hintsCfg.Enabled() { if len(bConfigs) > 0 { - return nil, errors.New("hints.enabled is incompatible with manually defining builders") + return Builders{}, errors.New("hints.enabled is incompatible with manually defining builders") } // pass rest of hints settings to the builder @@ -116,10 +126,10 @@ func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, for _, bcfg := range bConfigs { builder, err := Registry.BuildBuilder(bcfg) if err != nil { - return nil, err + return Builders{}, err } - builders = append(builders, builder) + builders.Builders = append(builders.Builders, builder) } - + builders.keystore = keystore return builders, nil } diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 75cc0dafaebf..804f8cb96394 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -61,11 +61,11 @@ func TestBuilderRegistry(t *testing.T) { // Try to create a config with fake builder and assert length // of configs returned is one - res := builder.CreateConfig(nil) + res := builder.CreateConfig(nil, nil) assert.Equal(t, len(res), 1) builders := Builders{} - builders = append(builders, builder) + builders.Builders = append(builders.Builders, builder) // Try using builders object for the same as above and expect // the same result diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index f2600195b8f7..8db0031138f0 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -85,7 +85,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs or hints defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, nil) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index 9fcb64823bf8..6495862f42c2 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -77,7 +77,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, nil) + builders, err := autodiscover.NewBuilders(config.Builders, nil, nil) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 2283cec77478..ac3e21ca40e0 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -58,7 +58,7 @@ type Provider struct { } // AutodiscoverBuilder builds and returns an autodiscover provider -func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore keystore.Keystore) (autodiscover.Provider, error) { +func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore keystore.Keystore) (autodiscover.Provider, error) { logger := logp.NewLogger("autodiscover") errWrap := func(err error) error { @@ -76,12 +76,16 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(err) } - mapper, err := template.NewConfigMapper(config.Templates, keystore) + mapper, err := template.NewConfigMapper(config.Templates, keyStore) if err != nil { return nil, errWrap(err) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) + k8sKeystore, err := kubernetes.CreateKubernetesKeystoreBackend(client) + if err != nil { + logger.Errorf("") + } + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, k8sKeystore) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index e1e560e7e9c6..4332efad3ade 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -77,8 +77,11 @@ func (e Event) GetValue(key string) (interface{}, error) { // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config - opts := []ucfg.Option{ - ucfg.Resolve(keystore.ResolverWrap(c.keystore)), + var opts []ucfg.Option + if c.keystore != nil { + opts = []ucfg.Option{ + ucfg.Resolve(keystore.ResolverWrap(c.keystore)), + } } for _, mapping := range c.ConditionMaps { // An empty condition matches everything diff --git a/libbeat/common/kubernetes/util.go b/libbeat/common/kubernetes/util.go index 470d07373eee..3b098bf8391a 100644 --- a/libbeat/common/kubernetes/util.go +++ b/libbeat/common/kubernetes/util.go @@ -19,6 +19,7 @@ package kubernetes import ( "fmt" + "github.com/elastic/beats/libbeat/keystore" "io/ioutil" "os" "strings" @@ -70,6 +71,15 @@ func IsInCluster(kubeconfig string) bool { return true } +func CreateKubernetesKeystoreBackend(client kubernetes.Interface) (keystore.Keystore, error) { + ns, err := inClusterNamespace() + if err != nil { + return nil, fmt.Errorf("kubernetes: Couldn't get namespace when beat is in cluster with error: %+v", err.Error()) + } + k8sKeystore, _ := keystore.Factoryk8s(ns, client) + return k8sKeystore, nil +} + // DiscoverKubernetesNode figures out the Kubernetes node to use. // If host is provided in the config use it directly. // If beat is deployed in k8s cluster, use hostname of pod which is pod name to query pod meta for node name. diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/keystore/kubernetes_keystore.go new file mode 100644 index 000000000000..380b13746607 --- /dev/null +++ b/libbeat/keystore/kubernetes_keystore.go @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + + "github.com/elastic/beats/libbeat/common" +) + +// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace. +type KubernetesSecretsKeystore struct { + namespace string + client k8s.Interface +} + +// Factoryk8s Create the right keystore with the configured options. +func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { + keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client) + return keystore, err +} + +// NewKubernetesSecretsKeystore returns an new k8s Keystore +func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { + keystore := KubernetesSecretsKeystore{ + namespace: keystoreNamespace, + client: ks8client, + } + return &keystore, nil +} + +// Retrieve return a SecureString instance that will contains both the key and the secret. +func (k *KubernetesSecretsKeystore) Retrieve(key string) (*SecureString, error) { + // key = "kubernetes:somenamespace:somesecret:value" + toks := strings.Split(key, ":") + ns := toks[1] + secretName := toks[2] + secretVar := toks[3] + if ns != k.namespace { + return nil, fmt.Errorf("cannot access Kubernetes secrets from a different namespace than: %v", ns) + } + secret, err := k.client.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + secretString := secret.Data[secretVar] + return NewSecureString(secretString), nil +} + +// GetConfig returns common.Config representation of the key / secret pair to be merged with other +// loaded configuration. +func (k *KubernetesSecretsKeystore) GetConfig() (*common.Config, error) { + return nil, nil +} + +// IsPersisted return if the keystore is physically persisted on disk. +func (k *KubernetesSecretsKeystore) IsPersisted() bool { + return true +} diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 8dc265bfce29..f70d9c618ffa 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -19,6 +19,7 @@ package hints import ( "fmt" + "github.com/elastic/go-ucfg" "strings" @@ -69,7 +70,7 @@ func NewMetricHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create configs based on hints passed from providers -func (m *metricHints) CreateConfig(event bus.Event) []*common.Config { +func (m *metricHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { var config []*common.Config host, _ := event["host"].(string) if host == "" { @@ -94,7 +95,7 @@ func (m *metricHints) CreateConfig(event bus.Event) []*common.Config { } logp.Debug("hints.builder", "generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, nil) + return template.ApplyConfigTemplate(event, configs, options) } @@ -155,7 +156,7 @@ func (m *metricHints) CreateConfig(event bus.Event) []*common.Config { // Apply information in event to the template to generate the final config // This especially helps in a scenario where endpoints are configured as: // co.elastic.metrics/hosts= "${data.host}:9090" - return template.ApplyConfigTemplate(event, config, nil) + return template.ApplyConfigTemplate(event, config, options) } func (m *metricHints) getModule(hints common.MapStr) string { diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index 4b3f7e0430b3..4968de0b0087 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -302,7 +302,7 @@ func TestGenerateHints(t *testing.T) { Key: defaultConfig().Key, Registry: mockRegister, } - cfgs := m.CreateConfig(test.event) + cfgs := m.CreateConfig(test.event, nil) assert.Equal(t, len(cfgs), test.len) if len(cfgs) != 0 { @@ -375,7 +375,7 @@ func TestGenerateHintsDoesNotAccessKeystore(t *testing.T) { Key: defaultConfig().Key, Registry: mockRegister, } - cfgs := m.CreateConfig(test.event) + cfgs := m.CreateConfig(test.event, nil) assert.Equal(t, len(cfgs), test.len) if len(cfgs) != 0 { config := common.MapStr{} From 4ddb15c3a0802a20899f19916f054d84e8fab027 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Apr 2020 17:36:37 +0300 Subject: [PATCH 03/18] Keystore access per namespace Signed-off-by: chrismark --- libbeat/autodiscover/builder.go | 35 +++++++++++++++---- .../providers/kubernetes/kubernetes.go | 6 +--- libbeat/common/kubernetes/util.go | 10 ------ 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index cb9cbeb47140..e63d33d0ef8d 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -20,12 +20,14 @@ package autodiscover import ( "errors" "fmt" - "github.com/elastic/beats/libbeat/keystore" - "github.com/elastic/go-ucfg" "strings" + "github.com/elastic/go-ucfg" + k8s "k8s.io/client-go/kubernetes" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/beats/v7/libbeat/keystore" ) // Builder provides an interface by which configs can be built from provider metadata @@ -37,7 +39,7 @@ type Builder interface { // Builders is a struct of Builder list objects and a common Keystore object type Builders struct { Builders []Builder - keystore keystore.Keystore + client k8s.Interface } // BuilderConstructor is a func used to generate a Builder object @@ -95,9 +97,11 @@ func (r *registry) BuildBuilder(c *common.Config) (Builder, error) { func (b Builders) GetConfig(event bus.Event) []*common.Config { configs := []*common.Config{} var opts []ucfg.Option - if b.keystore != nil { + + k8sKeystore := getK8sKeystore(event, b.client) + if k8sKeystore != nil { opts = []ucfg.Option{ - ucfg.Resolve(keystore.ResolverWrap(b.keystore)), + ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), } } for _, builder := range b.Builders { @@ -109,9 +113,26 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { return configs } +func getK8sKeystore(event bus.Event, client k8s.Interface) keystore.Keystore { + namespace := "" + if val, ok := event["kubernetes"]; ok { + kubernetesMeta := val.(common.MapStr) + ns, err := kubernetesMeta.GetValue("namespace") + if err != nil { + return nil + } + namespace = ns.(string) + } + if namespace != "" { + k8sKeystore, _ := keystore.Factoryk8s(namespace, client) + return k8sKeystore + } + return nil +} + // NewBuilders instances the given list of builders. hintsCfg holds `hints` settings // for simplified mode (single 'hints' builder) -func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, keystore keystore.Keystore) (Builders, error) { +func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, client k8s.Interface) (Builders, error) { var builders Builders if hintsCfg.Enabled() { if len(bConfigs) > 0 { @@ -130,6 +151,6 @@ func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, keystore ke } builders.Builders = append(builders.Builders, builder) } - builders.keystore = keystore + builders.client = client return builders, nil } diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index ac3e21ca40e0..87e3e3748afc 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -81,11 +81,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore return nil, errWrap(err) } - k8sKeystore, err := kubernetes.CreateKubernetesKeystoreBackend(client) - if err != nil { - logger.Errorf("") - } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, k8sKeystore) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, client) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/common/kubernetes/util.go b/libbeat/common/kubernetes/util.go index 3b098bf8391a..470d07373eee 100644 --- a/libbeat/common/kubernetes/util.go +++ b/libbeat/common/kubernetes/util.go @@ -19,7 +19,6 @@ package kubernetes import ( "fmt" - "github.com/elastic/beats/libbeat/keystore" "io/ioutil" "os" "strings" @@ -71,15 +70,6 @@ func IsInCluster(kubeconfig string) bool { return true } -func CreateKubernetesKeystoreBackend(client kubernetes.Interface) (keystore.Keystore, error) { - ns, err := inClusterNamespace() - if err != nil { - return nil, fmt.Errorf("kubernetes: Couldn't get namespace when beat is in cluster with error: %+v", err.Error()) - } - k8sKeystore, _ := keystore.Factoryk8s(ns, client) - return k8sKeystore, nil -} - // DiscoverKubernetesNode figures out the Kubernetes node to use. // If host is provided in the config use it directly. // If beat is deployed in k8s cluster, use hostname of pod which is pod name to query pod meta for node name. From c68fe18e2ba648bb219bab78e522a2c921caf48b Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 30 Apr 2020 13:29:39 +0300 Subject: [PATCH 04/18] Add k8s Keystore for static configs too Signed-off-by: chrismark --- libbeat/autodiscover/builder.go | 35 +++----- .../autodiscover/providers/docker/docker.go | 2 +- .../autodiscover/providers/jolokia/jolokia.go | 2 +- .../providers/kubernetes/kubernetes.go | 6 +- libbeat/autodiscover/template/config.go | 14 ++++ libbeat/autodiscover/template/config_test.go | 2 +- libbeat/keystore/kubernetes_keystore.go | 82 +++++++++++++++++++ 7 files changed, 116 insertions(+), 27 deletions(-) diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index e63d33d0ef8d..f24a949de8e1 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/elastic/go-ucfg" - k8s "k8s.io/client-go/kubernetes" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" @@ -39,7 +38,7 @@ type Builder interface { // Builders is a struct of Builder list objects and a common Keystore object type Builders struct { Builders []Builder - client k8s.Interface + kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry } // BuilderConstructor is a func used to generate a Builder object @@ -98,10 +97,12 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { configs := []*common.Config{} var opts []ucfg.Option - k8sKeystore := getK8sKeystore(event, b.client) - if k8sKeystore != nil { - opts = []ucfg.Option{ - ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), + if b.kubernetesKeystoresRegistry != nil { + k8sKeystore := b.kubernetesKeystoresRegistry.GetK8sKeystore(event) + if k8sKeystore != nil { + opts = []ucfg.Option{ + ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), + } } } for _, builder := range b.Builders { @@ -113,26 +114,14 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { return configs } -func getK8sKeystore(event bus.Event, client k8s.Interface) keystore.Keystore { - namespace := "" - if val, ok := event["kubernetes"]; ok { - kubernetesMeta := val.(common.MapStr) - ns, err := kubernetesMeta.GetValue("namespace") - if err != nil { - return nil - } - namespace = ns.(string) - } - if namespace != "" { - k8sKeystore, _ := keystore.Factoryk8s(namespace, client) - return k8sKeystore - } - return nil +// GetValue extracts given key from an Event +func (b Builders) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { + b.kubernetesKeystoresRegistry = k8sKeystoresRegistry } // NewBuilders instances the given list of builders. hintsCfg holds `hints` settings // for simplified mode (single 'hints' builder) -func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, client k8s.Interface) (Builders, error) { +func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) { var builders Builders if hintsCfg.Enabled() { if len(bConfigs) > 0 { @@ -151,6 +140,6 @@ func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config, client k8s. } builders.Builders = append(builders.Builders, builder) } - builders.client = client + return builders, nil } diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 8db0031138f0..f2600195b8f7 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -85,7 +85,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs or hints defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, nil) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index 6495862f42c2..9fcb64823bf8 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -77,7 +77,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, nil, nil) + builders, err := autodiscover.NewBuilders(config.Builders, nil) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 87e3e3748afc..c62b8b3f2d8c 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -81,7 +81,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore return nil, errWrap(err) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, client) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) if err != nil { return nil, errWrap(err) } @@ -91,6 +91,10 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore return nil, errWrap(err) } + k8sKeystoresRegistry := keystore.NewKubernetesKeystoresRegistry(logger, client) + mapper.SetKubernetesKeystoresRegistry(k8sKeystoresRegistry) + builders.SetKubernetesKeystoresRegistry(k8sKeystoresRegistry) + p := &Provider{ config: config, bus: bus, diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 4332efad3ade..d3fc85ebd43a 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -31,6 +31,7 @@ import ( type Mapper struct { ConditionMaps []*ConditionMap keystore keystore.Keystore + kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry } // ConditionMap maps a condition to the configs to use when it's triggered @@ -74,6 +75,11 @@ func (e Event) GetValue(key string) (interface{}, error) { return val, nil } +// GetValue extracts given key from an Event +func (c Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { + c.kubernetesKeystoresRegistry = k8sKeystoresRegistry +} + // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config @@ -83,6 +89,14 @@ func (c Mapper) GetConfig(event bus.Event) []*common.Config { ucfg.Resolve(keystore.ResolverWrap(c.keystore)), } } + if c.kubernetesKeystoresRegistry != nil { + k8sKeystore := c.kubernetesKeystoresRegistry.GetK8sKeystore(event) + if k8sKeystore != nil { + opts = []ucfg.Option{ + ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), + } + } + } for _, mapping := range c.ConditionMaps { // An empty condition matches everything conditionOk := mapping.Condition == nil || mapping.Condition.Check(Event(event)) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index f2bf33bf3dbb..d0f1957ab833 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -87,7 +87,7 @@ func TestConfigsMapping(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings, nil) + mapper, err := NewConfigMapper(mappings, nil, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/keystore/kubernetes_keystore.go index 380b13746607..bda0c04a66d1 100644 --- a/libbeat/keystore/kubernetes_keystore.go +++ b/libbeat/keystore/kubernetes_keystore.go @@ -19,6 +19,8 @@ package keystore import ( "fmt" + "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/libbeat/logp" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,6 +29,14 @@ import ( "github.com/elastic/beats/libbeat/common" ) +type KubernetesKeystores map[string]Keystore + +type KubernetesKeystoresRegistry struct { + kubernetesKeystores KubernetesKeystores + logger *logp.Logger + client k8s.Interface +} + // KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace. type KubernetesSecretsKeystore struct { namespace string @@ -39,6 +49,47 @@ func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface) (Keystore, er return keystore, err } +func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) *KubernetesKeystoresRegistry { + return &KubernetesKeystoresRegistry{ + kubernetesKeystores: KubernetesKeystores{}, + logger: logger, + client: client, + } +} + +// GetK8sKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace of creates a new one. +func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore { + namespace := "" + if val, ok := event["kubernetes"]; ok { + kubernetesMeta := val.(common.MapStr) + ns, err := kubernetesMeta.GetValue("namespace") + if err != nil { + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil + } + namespace = ns.(string) + } + if namespace != "" { + // either retrieve already stored keystore or create a new one for the namespace + storedKeystore := kr.lookupForKeystore(namespace) + if storedKeystore != nil { + return storedKeystore + } + k8sKeystore, _ := Factoryk8s(namespace, kr.client) + kr.kubernetesKeystores["namespace"] = k8sKeystore + return k8sKeystore + } + return nil +} + +func (kr *KubernetesKeystoresRegistry) lookupForKeystore(keystoreNamespace string) (Keystore) { + if keystore, ok := kr.kubernetesKeystores[keystoreNamespace]; ok { + return keystore + } + return nil +} + + // NewKubernetesSecretsKeystore returns an new k8s Keystore func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { keystore := KubernetesSecretsKeystore{ @@ -76,3 +127,34 @@ func (k *KubernetesSecretsKeystore) GetConfig() (*common.Config, error) { func (k *KubernetesSecretsKeystore) IsPersisted() bool { return true } + +func (b Builders) getK8sKeystore(event bus.Event, client k8s.Interface, k8sKeystores map[string]keystore.Keystore) keystore.Keystore { + namespace := "" + if val, ok := event["kubernetes"]; ok { + kubernetesMeta := val.(common.MapStr) + ns, err := kubernetesMeta.GetValue("namespace") + if err != nil { + b.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil + } + namespace = ns.(string) + } + if namespace != "" { + // either retrieve already stored keystore or create a new one for the namespace + storedKeystore := b.lookupForKeystore(namespace) + if storedKeystore != nil { + return storedKeystore + } + k8sKeystore, _ := keystore.Factoryk8s(namespace, client) + b.k8sKeystores["namespace"] = k8sKeystore + return k8sKeystore + } + return nil +} + +func (b Builders) lookupForKeystore(keystoreNamespace string) (keystore.Keystore) { + if keystore, ok := b.k8sKeystores[keystoreNamespace]; ok { + return keystore + } + return nil +} From d0f3629e988bce17feddf73e7a49586ced155bac Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 30 Apr 2020 18:58:02 +0300 Subject: [PATCH 05/18] working poc Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/metricbeat-kubernetes.yaml | 1 + .../metricbeat/metricbeat-role.yaml | 1 + filebeat/autodiscover/builder/hints/logs.go | 3 +- libbeat/autodiscover/autodiscover.go | 4 ++ libbeat/autodiscover/builder.go | 6 +- libbeat/autodiscover/template/config.go | 9 +-- libbeat/keystore/kubernetes_keystore.go | 63 +++++++------------ .../autodiscover/builder/hints/metrics.go | 4 +- 9 files changed, 40 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bcb358e8ef16..3d437960fb81 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464] - Add keystore support for autodiscover static configurations. {pull]16306[16306] - Add Kerberos support to Elasticsearch output. {pull}17927[17927] +- Add k8s keystore backend. {pull}18096[18096] *Auditbeat* diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 179f33089e18..216bbc24f9fb 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -335,6 +335,7 @@ rules: - namespaces - events - pods + - secrets verbs: ["get", "list", "watch"] - apiGroups: ["extensions"] resources: diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml index 55cea94fb3c1..ba9bff7c28fc 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -11,6 +11,7 @@ rules: - namespaces - events - pods + - secrets verbs: ["get", "list", "watch"] - apiGroups: ["extensions"] resources: diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 33c90a239252..537c54047b88 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -19,9 +19,10 @@ package hints import ( "fmt" - "github.com/elastic/go-ucfg" "regexp" + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/filebeat/fileset" "github.com/elastic/beats/v7/filebeat/harvester" "github.com/elastic/beats/v7/libbeat/autodiscover" diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index 668a350b865c..b41c2e076bc4 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -193,6 +193,10 @@ func (a *Autodiscover) handleStart(event bus.Event) bool { if a.logger.IsDebug() { for _, c := range configs { + // TODO remove the non private log before merging + rc := map[string]interface{}{} + c.Unpack(&rc) + a.logger.Debugf("Generated config: %+v", rc) a.logger.Debugf("Generated config: %+v", common.DebugString(c, true)) } } diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index f24a949de8e1..f85924361dd7 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -37,7 +37,7 @@ type Builder interface { // Builders is a struct of Builder list objects and a common Keystore object type Builders struct { - Builders []Builder + Builders []Builder kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry } @@ -95,7 +95,7 @@ func (r *registry) BuildBuilder(c *common.Config) (Builder, error) { // GetConfig creates configs for all builders initialized. func (b Builders) GetConfig(event bus.Event) []*common.Config { configs := []*common.Config{} - var opts []ucfg.Option + var opts []ucfg.Option if b.kubernetesKeystoresRegistry != nil { k8sKeystore := b.kubernetesKeystoresRegistry.GetK8sKeystore(event) @@ -115,7 +115,7 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { } // GetValue extracts given key from an Event -func (b Builders) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { +func (b *Builders) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { b.kubernetesKeystoresRegistry = k8sKeystoresRegistry } diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index d3fc85ebd43a..5da9b96fb1c8 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -29,8 +29,8 @@ import ( // Mapper maps config templates with conditions, if a match happens on a discover event // the given template will be used as config type Mapper struct { - ConditionMaps []*ConditionMap - keystore keystore.Keystore + ConditionMaps []*ConditionMap + keystore keystore.Keystore kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry } @@ -76,14 +76,14 @@ func (e Event) GetValue(key string) (interface{}, error) { } // GetValue extracts given key from an Event -func (c Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { +func (c *Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { c.kubernetesKeystoresRegistry = k8sKeystoresRegistry } // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config - var opts []ucfg.Option + var opts []ucfg.Option if c.keystore != nil { opts = []ucfg.Option{ ucfg.Resolve(keystore.ResolverWrap(c.keystore)), @@ -129,6 +129,7 @@ func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options []uc ucfg.VarExp, } opts = append(opts, options...) + logp.Err("Here are the options %v", options) for _, config := range configs { c, err := ucfg.NewFrom(config, opts...) diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/keystore/kubernetes_keystore.go index bda0c04a66d1..9e944e6c9275 100644 --- a/libbeat/keystore/kubernetes_keystore.go +++ b/libbeat/keystore/kubernetes_keystore.go @@ -19,22 +19,22 @@ package keystore import ( "fmt" - "github.com/elastic/beats/libbeat/common/bus" - "github.com/elastic/beats/libbeat/logp" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/beats/v7/libbeat/logp" ) type KubernetesKeystores map[string]Keystore type KubernetesKeystoresRegistry struct { kubernetesKeystores KubernetesKeystores - logger *logp.Logger - client k8s.Interface + logger *logp.Logger + client k8s.Interface } // KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace. @@ -52,8 +52,8 @@ func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface) (Keystore, er func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) *KubernetesKeystoresRegistry { return &KubernetesKeystoresRegistry{ kubernetesKeystores: KubernetesKeystores{}, - logger: logger, - client: client, + logger: logger, + client: client, } } @@ -82,14 +82,13 @@ func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore return nil } -func (kr *KubernetesKeystoresRegistry) lookupForKeystore(keystoreNamespace string) (Keystore) { +func (kr *KubernetesKeystoresRegistry) lookupForKeystore(keystoreNamespace string) Keystore { if keystore, ok := kr.kubernetesKeystores[keystoreNamespace]; ok { return keystore } return nil } - // NewKubernetesSecretsKeystore returns an new k8s Keystore func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { keystore := KubernetesSecretsKeystore{ @@ -102,15 +101,26 @@ func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interf // Retrieve return a SecureString instance that will contains both the key and the secret. func (k *KubernetesSecretsKeystore) Retrieve(key string) (*SecureString, error) { // key = "kubernetes:somenamespace:somesecret:value" - toks := strings.Split(key, ":") + toks := strings.Split(key, ".") ns := toks[1] secretName := toks[2] secretVar := toks[3] if ns != k.namespace { return nil, fmt.Errorf("cannot access Kubernetes secrets from a different namespace than: %v", ns) } - secret, err := k.client.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) + secretIntefrace := k.client.CoreV1().Secrets(ns) + secrets, err := secretIntefrace.List(metav1.ListOptions{}) + if err != nil { + // log the error here + return nil, err + } + if len(secrets.Items) == 0 { + // log the error here + return nil, fmt.Errorf("no secrets found for namespace: %v", ns) + } + secret, err := secretIntefrace.Get(secretName, metav1.GetOptions{}) if err != nil { + // log the error here return nil, err } secretString := secret.Data[secretVar] @@ -127,34 +137,3 @@ func (k *KubernetesSecretsKeystore) GetConfig() (*common.Config, error) { func (k *KubernetesSecretsKeystore) IsPersisted() bool { return true } - -func (b Builders) getK8sKeystore(event bus.Event, client k8s.Interface, k8sKeystores map[string]keystore.Keystore) keystore.Keystore { - namespace := "" - if val, ok := event["kubernetes"]; ok { - kubernetesMeta := val.(common.MapStr) - ns, err := kubernetesMeta.GetValue("namespace") - if err != nil { - b.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) - return nil - } - namespace = ns.(string) - } - if namespace != "" { - // either retrieve already stored keystore or create a new one for the namespace - storedKeystore := b.lookupForKeystore(namespace) - if storedKeystore != nil { - return storedKeystore - } - k8sKeystore, _ := keystore.Factoryk8s(namespace, client) - b.k8sKeystores["namespace"] = k8sKeystore - return k8sKeystore - } - return nil -} - -func (b Builders) lookupForKeystore(keystoreNamespace string) (keystore.Keystore) { - if keystore, ok := b.k8sKeystores[keystoreNamespace]; ok { - return keystore - } - return nil -} diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index f70d9c618ffa..cf5927e98d74 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -19,10 +19,10 @@ package hints import ( "fmt" - "github.com/elastic/go-ucfg" - "strings" + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" From c92b6e5eb7b02f4d6193ff5dc4cbdd7132de56e4 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 6 May 2020 15:05:26 +0300 Subject: [PATCH 06/18] Add error handling Signed-off-by: chrismark --- libbeat/autodiscover/template/config_test.go | 2 +- libbeat/keystore/kubernetes_keystore.go | 47 ++++++++++++-------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index d0f1957ab833..f2bf33bf3dbb 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -87,7 +87,7 @@ func TestConfigsMapping(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings, nil, nil) + mapper, err := NewConfigMapper(mappings, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/keystore/kubernetes_keystore.go index 9e944e6c9275..a5b71409e613 100644 --- a/libbeat/keystore/kubernetes_keystore.go +++ b/libbeat/keystore/kubernetes_keystore.go @@ -18,7 +18,6 @@ package keystore import ( - "fmt" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,11 +40,12 @@ type KubernetesKeystoresRegistry struct { type KubernetesSecretsKeystore struct { namespace string client k8s.Interface + logger *logp.Logger } // Factoryk8s Create the right keystore with the configured options. -func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { - keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client) +func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (Keystore, error) { + keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) return keystore, err } @@ -57,7 +57,7 @@ func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) * } } -// GetK8sKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace of creates a new one. +// GetK8sKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore { namespace := "" if val, ok := event["kubernetes"]; ok { @@ -75,10 +75,11 @@ func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore if storedKeystore != nil { return storedKeystore } - k8sKeystore, _ := Factoryk8s(namespace, kr.client) + k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger) kr.kubernetesKeystores["namespace"] = k8sKeystore return k8sKeystore } + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) return nil } @@ -90,38 +91,48 @@ func (kr *KubernetesKeystoresRegistry) lookupForKeystore(keystoreNamespace strin } // NewKubernetesSecretsKeystore returns an new k8s Keystore -func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface) (Keystore, error) { +func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (Keystore, error) { keystore := KubernetesSecretsKeystore{ namespace: keystoreNamespace, client: ks8client, + logger: logger, } return &keystore, nil } // Retrieve return a SecureString instance that will contains both the key and the secret. func (k *KubernetesSecretsKeystore) Retrieve(key string) (*SecureString, error) { - // key = "kubernetes:somenamespace:somesecret:value" - toks := strings.Split(key, ".") - ns := toks[1] - secretName := toks[2] - secretVar := toks[3] + // key = "kubernetes.somenamespace.somesecret.value" + tokens := strings.Split(key, ".") + if len(tokens) != 4 { + k.logger.Debugf( + "not valid secret key: %v. Secrets should be of the following format %v", + key, + "kubernetes.somenamespace.somesecret.value", + ) + return nil, ErrKeyDoesntExists + } + ns := tokens[1] + secretName := tokens[2] + secretVar := tokens[3] if ns != k.namespace { - return nil, fmt.Errorf("cannot access Kubernetes secrets from a different namespace than: %v", ns) + k.logger.Debugf("cannot access Kubernetes secrets from a different namespace than: %v", ns) + return nil, ErrKeyDoesntExists } secretIntefrace := k.client.CoreV1().Secrets(ns) secrets, err := secretIntefrace.List(metav1.ListOptions{}) if err != nil { - // log the error here - return nil, err + k.logger.Errorf("Could not retrieve secrets from k8s API: %v", err) + return nil, ErrKeyDoesntExists } if len(secrets.Items) == 0 { - // log the error here - return nil, fmt.Errorf("no secrets found for namespace: %v", ns) + k.logger.Debugf("no secrets found for namespace: %v", ns) + return nil, ErrKeyDoesntExists } secret, err := secretIntefrace.Get(secretName, metav1.GetOptions{}) if err != nil { - // log the error here - return nil, err + k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) + return nil, ErrKeyDoesntExists } secretString := secret.Data[secretVar] return NewSecureString(secretString), nil From c5fd9e83687fd302f3e755dbbb39322ddc5aad85 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 6 May 2020 16:14:50 +0300 Subject: [PATCH 07/18] Fix keystore override issue Signed-off-by: chrismark --- libbeat/autodiscover/template/config.go | 10 +++------- metricbeat/autodiscover/builder/hints/metrics_test.go | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 5da9b96fb1c8..512865786f99 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -83,18 +83,14 @@ func (c *Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.K // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config - var opts []ucfg.Option + opts := []ucfg.Option{} if c.keystore != nil { - opts = []ucfg.Option{ - ucfg.Resolve(keystore.ResolverWrap(c.keystore)), - } + opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(c.keystore))) } if c.kubernetesKeystoresRegistry != nil { k8sKeystore := c.kubernetesKeystoresRegistry.GetK8sKeystore(event) if k8sKeystore != nil { - opts = []ucfg.Option{ - ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), - } + opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(k8sKeystore))) } } for _, mapping := range c.ConditionMaps { diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index 4968de0b0087..f0488ecaa73c 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -328,7 +328,7 @@ func TestGenerateHints(t *testing.T) { } } -func TestGenerateHintsDoesNotAccessKeystore(t *testing.T) { +func TestGenerateHintsDoesNotAccessGlobalKeystore(t *testing.T) { path := getTemporaryKeystoreFile() defer os.Remove(path) // store the secret From 1c4166ff937b1cbc8bf041a5409fd780ca70d19f Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 6 May 2020 16:18:00 +0300 Subject: [PATCH 08/18] Fix tests Signed-off-by: chrismark --- libbeat/autodiscover/providers/kubernetes/config_test.go | 3 ++- libbeat/autodiscover/template/config.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index 55fd601037dd..6b8771ab213b 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -20,6 +20,7 @@ package kubernetes import ( "testing" + "github.com/elastic/go-ucfg" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/autodiscover" @@ -76,6 +77,6 @@ func newMockBuilder(_ *common.Config) (autodiscover.Builder, error) { return &mockBuilder{}, nil } -func (m *mockBuilder) CreateConfig(event bus.Event) []*common.Config { +func (m *mockBuilder) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { return nil } diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 512865786f99..91ee2c460485 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -18,12 +18,13 @@ package template import ( + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/conditions" "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/go-ucfg" ) // Mapper maps config templates with conditions, if a match happens on a discover event From a069dd7915d587bbc1a5182d964767158d4b22d7 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 6 May 2020 16:38:27 +0300 Subject: [PATCH 09/18] Fix monitors Signed-off-by: chrismark --- heartbeat/autodiscover/builder/hints/monitors.go | 4 +++- heartbeat/autodiscover/builder/hints/monitors_test.go | 2 +- libbeat/autodiscover/builder_test.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 431dbe916344..39bfcd6623e0 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -23,6 +23,8 @@ import ( "strconv" "strings" + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" @@ -60,7 +62,7 @@ func NewHeartbeatHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create config based on input hints in the bus event -func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config { +func (hb *heartbeatHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { var hints common.MapStr hIface, ok := event["hints"] if ok { diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index 82c4dc6262c8..e5ed7ec44910 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -203,7 +203,7 @@ func TestGenerateHints(t *testing.T) { config: defaultConfig(), logger: logp.NewLogger("hints.builder"), } - cfgs := m.CreateConfig(test.event) + cfgs := m.CreateConfig(test.event, nil) assert.Equal(t, len(cfgs), test.len, test.message) if len(cfgs) != 0 { diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 804f8cb96394..9a3c9f0e6893 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -20,6 +20,7 @@ package autodiscover import ( "testing" + "github.com/elastic/go-ucfg" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" @@ -28,7 +29,7 @@ import ( type fakeBuilder struct{} -func (f *fakeBuilder) CreateConfig(event bus.Event) []*common.Config { +func (f *fakeBuilder) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { return []*common.Config{common.NewConfig()} } From c47cf048466e96609829f0d46e5674a38a07f5a3 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 6 May 2020 17:11:10 +0300 Subject: [PATCH 10/18] fmt code Signed-off-by: chrismark --- libbeat/autodiscover/builder_test.go | 3 ++- libbeat/autodiscover/providers/kubernetes/config_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 9a3c9f0e6893..9df81aa63d98 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -20,9 +20,10 @@ package autodiscover import ( "testing" - "github.com/elastic/go-ucfg" "github.com/stretchr/testify/assert" + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" ) diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index 6b8771ab213b..4f0898f4f6aa 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -20,9 +20,10 @@ package kubernetes import ( "testing" - "github.com/elastic/go-ucfg" "github.com/stretchr/testify/assert" + "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" From 05e3ed08748f57c249963ace051ad55859807d61 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 7 May 2020 10:22:28 +0300 Subject: [PATCH 11/18] cleanups Signed-off-by: chrismark --- libbeat/autodiscover/autodiscover.go | 5 ----- libbeat/autodiscover/builder.go | 5 +++-- libbeat/autodiscover/template/config.go | 9 +++++---- libbeat/keystore/kubernetes_keystore.go | 7 +++++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index b41c2e076bc4..5235ebbe7c1e 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -191,12 +191,7 @@ func (a *Autodiscover) handleStart(event bus.Event) bool { } if a.logger.IsDebug() { - for _, c := range configs { - // TODO remove the non private log before merging - rc := map[string]interface{}{} - c.Unpack(&rc) - a.logger.Debugf("Generated config: %+v", rc) a.logger.Debugf("Generated config: %+v", common.DebugString(c, true)) } } diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index f85924361dd7..69eedb80ad0f 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -35,7 +35,8 @@ type Builder interface { CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config } -// Builders is a struct of Builder list objects and a common Keystore object +// Builders is a struct of Builder list objects and a `kubernetesKeystoresRegistry` object, which +// holds Kubernetes keystores for known namespaces type Builders struct { Builders []Builder kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry @@ -114,7 +115,7 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { return configs } -// GetValue extracts given key from an Event +// SetKubernetesKeystoresRegistry set the k8sKeystoresRegistry of the Builders object func (b *Builders) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { b.kubernetesKeystoresRegistry = k8sKeystoresRegistry } diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 91ee2c460485..002b09672c41 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -27,8 +27,10 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -// Mapper maps config templates with conditions, if a match happens on a discover event -// the given template will be used as config +// Mapper maps config templates with conditions in ConditionMaps, if a match happens on a discover event +// the given template will be used as config. +// Mapper also includes the global Keystore object at `keystore` and `kubernetesKeystoresRegistry`, which +// holds Kubernetes keystores for known namespaces type Mapper struct { ConditionMaps []*ConditionMap keystore keystore.Keystore @@ -76,7 +78,7 @@ func (e Event) GetValue(key string) (interface{}, error) { return val, nil } -// GetValue extracts given key from an Event +// SetKubernetesKeystoresRegistry set the k8sKeystoresRegistry of the Mapper object func (c *Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { c.kubernetesKeystoresRegistry = k8sKeystoresRegistry } @@ -126,7 +128,6 @@ func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options []uc ucfg.VarExp, } opts = append(opts, options...) - logp.Err("Here are the options %v", options) for _, config := range configs { c, err := ucfg.NewFrom(config, opts...) diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/keystore/kubernetes_keystore.go index a5b71409e613..5d4cab7c7a16 100644 --- a/libbeat/keystore/kubernetes_keystore.go +++ b/libbeat/keystore/kubernetes_keystore.go @@ -30,25 +30,28 @@ import ( type KubernetesKeystores map[string]Keystore +// KubernetesKeystoresRegistry holds KubernetesKeystores for known namespaces. Once a Keystore for one k8s namespace +// is initialized it will be reused every time it is needed. type KubernetesKeystoresRegistry struct { kubernetesKeystores KubernetesKeystores logger *logp.Logger client k8s.Interface } -// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace. +// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace type KubernetesSecretsKeystore struct { namespace string client k8s.Interface logger *logp.Logger } -// Factoryk8s Create the right keystore with the configured options. +// Factoryk8s Create the right keystore with the configured options func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (Keystore, error) { keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) return keystore, err } +// NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) *KubernetesKeystoresRegistry { return &KubernetesKeystoresRegistry{ kubernetesKeystores: KubernetesKeystores{}, From 5e76432ac764cb40e63bd4e07ef050faaf7924ba Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 25 May 2020 18:24:18 +0300 Subject: [PATCH 12/18] Add docs Signed-off-by: chrismark --- libbeat/docs/shared-autodiscover.asciidoc | 16 ----- metricbeat/docs/autodiscover-hints.asciidoc | 3 +- .../autodiscover-kubernetes-config.asciidoc | 70 ++++++++++++++++++- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 5279b6f3ea0d..5da68216b0e3 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -244,22 +244,6 @@ running configuration for a container, 60s by default. include::../../{beatname_lc}/docs/autodiscover-kubernetes-config.asciidoc[] -[float] -===== Manually Defining Ports with Kubernetes - -Declare exposed ports in your pod spec if possible. Otherwise, you will need to use -multiple templates with complex filtering rules. The {port} variable will not be -present, and you will need to hardcode ports. Example: `{data.host}:1234` - -When ports are not declared, Autodiscover generates a config using your provided -template once per pod, and once per container. These generated configs are -de-duplicated after they are generated. If the generated configs for multiple -containers are identical, they will be merged into one config. - -Pods share an identical host. If only the `{data.host}` variable is interpolated, -then one config will be generated per host. The configs will be identical. -After they are de-duplicated, only one will be used. - ifdef::autodiscoverJolokia[] [float] ===== Jolokia diff --git a/metricbeat/docs/autodiscover-hints.asciidoc b/metricbeat/docs/autodiscover-hints.asciidoc index a34b623bd367..25029a61d05d 100644 --- a/metricbeat/docs/autodiscover-hints.asciidoc +++ b/metricbeat/docs/autodiscover-hints.asciidoc @@ -45,7 +45,8 @@ The username to use for authentication The password to use for authentication. It is recommended to retrieve this sensitive information from an ENV variable and avoid placing passwords in plain text. Unlike static autodiscover configuration, hints based autodiscover has -no access to the keystore of Metricbeat since it could be a potential security issue. +no access to the keystore of Metricbeat since it could be a potential security issue. However hints based autodiscover +can make use of Kuberentes Secrets as described in <>. [float] ===== `co.elastic.metrics/ssl.*` diff --git a/metricbeat/docs/autodiscover-kubernetes-config.asciidoc b/metricbeat/docs/autodiscover-kubernetes-config.asciidoc index f3e6a74cdfbf..a18993dd3cd6 100644 --- a/metricbeat/docs/autodiscover-kubernetes-config.asciidoc +++ b/metricbeat/docs/autodiscover-kubernetes-config.asciidoc @@ -18,7 +18,29 @@ metricbeat.autodiscover: This configuration launches a `prometheus` module for all containers of pods annotated `prometheus.io/scrape=true`. -Also Metricbeat autodiscover supports leveraging <> in order to retrieve sensitive data like passwords. +[float] +===== Manually Defining Ports with Kubernetes + +Declare exposed ports in your pod spec if possible. Otherwise, you will need to use +multiple templates with complex filtering rules. The {port} variable will not be +present, and you will need to hardcode ports. Example: `{data.host}:1234` + +When ports are not declared, Autodiscover generates a config using your provided +template once per pod, and once per container. These generated configs are +de-duplicated after they are generated. If the generated configs for multiple +containers are identical, they will be merged into one config. + +Pods share an identical host. If only the `{data.host}` variable is interpolated, +then one config will be generated per host. The configs will be identical. +After they are de-duplicated, only one will be used. + +[float] +[[kubernetes-secrets]] +===== Metricbeat Autodiscover Secret Management + +[float] +====== Local Keystore +Metricbeat autodiscover supports leveraging <> in order to retrieve sensitive data like passwords. Here is an example of how a configuration using keystore would look like: ["source","yaml",subs="attributes"] @@ -38,3 +60,49 @@ metricbeat.autodiscover: ------------------------------------------------------------------------------------- where `REDIS_PASSWORD` is a key stored in local keystore of Metricbeat. + +[float] +===== Kubernetes Secrets +Metricbeat autodiscover supports leveraging https://kubernetes.io/docs/concepts/configuration/secret/[Kubernetes secrets] +in order to retrieve sensitive data like passwords. +Here is an example of how a configuration using Kubernetes secrets would look like: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +metricbeat.autodiscover: + providers: + - type: kubernetes + templates: + - condition: + contains: + kubernetes.labels.app: "redis" + config: + - module: redis + metricsets: ["info", "keyspace"] + hosts: "${data.host}:6379" + password: "${kubernetes.default.somesecret.value}" +------------------------------------------------------------------------------------- + +where `kubernetes.default.somesecret.value` specifies a key stored as Kubernetes secret as following: + +. Kubernetes Namespace: `default` +. Kubernetes Secret Name: `somesecret` +. Secret Data Key: `value` + +This secret can be created in a Kubernetes environment using the following the command: +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +cat << EOF | kubectl apply -f - +apiVersion: v1 +kind: Secret +metadata: + name: somesecret +type: Opaque +data: + value: $(echo -n "passpass" | base64) +EOF +------------------------------------------------------------------------------------- + + +Note that Pods can only consume secrets that belong to the same Kubernetes namespace. For instance if Pod `my-redis` +is running under `staging` namespace, it cannot access a secret under `testing` namespace for example `kubernetes.testing.xxx.yyy`. From a97506557816b28f105fcb1b734ac82fa2599f47 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 26 May 2020 12:26:43 +0300 Subject: [PATCH 13/18] review fixes Signed-off-by: chrismark --- filebeat/autodiscover/builder/hints/logs.go | 6 +-- .../autodiscover/builder/hints/logs_test.go | 4 +- .../autodiscover/builder/hints/monitors.go | 6 +-- .../autodiscover/appenders/config/config.go | 2 +- libbeat/autodiscover/builder.go | 31 +++++++-------- libbeat/autodiscover/builder_test.go | 4 +- .../autodiscover/providers/docker/docker.go | 4 +- .../autodiscover/providers/jolokia/jolokia.go | 4 +- .../providers/kubernetes/config_test.go | 2 +- .../providers/kubernetes/kubernetes.go | 15 ++++--- .../providers/kubernetes/node_test.go | 2 +- .../providers/kubernetes/pod_test.go | 2 +- .../providers/kubernetes/service_test.go | 2 +- libbeat/autodiscover/template/config.go | 30 +++++++------- libbeat/autodiscover/template/config_test.go | 6 +-- .../kubernetes_keystore.go | 39 ++++++++----------- libbeat/keystore/keystore.go | 8 +++- .../autodiscover/builder/hints/metrics.go | 6 +-- .../builder/hints/metrics_test.go | 4 +- .../providers/aws/ec2/provider.go | 2 +- .../providers/aws/elb/provider.go | 2 +- 21 files changed, 89 insertions(+), 92 deletions(-) rename libbeat/{keystore => k8skeystore}/kubernetes_keystore.go (81%) diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 537c54047b88..70758a8a0280 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -72,7 +72,7 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create config based on input hints in the bus event -func (l *logHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { +func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { var hints common.MapStr hIface, ok := event["hints"] if ok { @@ -111,7 +111,7 @@ func (l *logHints) CreateConfig(event bus.Event, options []ucfg.Option) []*commo } logp.Debug("hints.builder", "generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, nil) + return template.ApplyConfigTemplate(event, configs) } tempCfg := common.MapStr{} @@ -165,7 +165,7 @@ func (l *logHints) CreateConfig(event bus.Event, options []ucfg.Option) []*commo logp.Debug("hints.builder", "generated config %+v", config) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, []*common.Config{config}, nil) + return template.ApplyConfigTemplate(event, []*common.Config{config}) } func (l *logHints) getMultiline(hints common.MapStr) common.MapStr { diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 7959431bc671..b316cdb506c2 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -628,7 +628,7 @@ func TestGenerateHints(t *testing.T) { t.Fatal(err) } - cfgs := l.CreateConfig(test.event, nil) + cfgs := l.CreateConfig(test.event) assert.Equal(t, len(cfgs), test.len, test.msg) if test.len != 0 { config := common.MapStr{} @@ -860,7 +860,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { t.Fatal(err) } - cfgs := l.CreateConfig(test.event, nil) + cfgs := l.CreateConfig(test.event) assert.Equal(t, len(cfgs), test.len, test.msg) if test.len != 0 { config := common.MapStr{} diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 39bfcd6623e0..f9fe8847d3ec 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -62,7 +62,7 @@ func NewHeartbeatHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create config based on input hints in the bus event -func (hb *heartbeatHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { +func (hb *heartbeatHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { var hints common.MapStr hIface, ok := event["hints"] if ok { @@ -93,7 +93,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event, options []ucfg.Option) [ } hb.logger.Debugf("generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, nil) + return template.ApplyConfigTemplate(event, configs) } tempCfg := common.MapStr{} @@ -123,7 +123,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event, options []ucfg.Option) [ } // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, nil) + return template.ApplyConfigTemplate(event, configs) } func (hb *heartbeatHints) getType(hints common.MapStr) common.MapStr { diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 156fcdf07b26..018ee1b587d8 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -104,7 +104,7 @@ func (c *configAppender) Append(event bus.Event) { } // Apply the template - template.ApplyConfigTemplate(event, cfgs, nil) + template.ApplyConfigTemplate(event, cfgs) } // Replace old config with newly appended configs diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index 69eedb80ad0f..0f57b187ea05 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -32,14 +32,14 @@ import ( // Builder provides an interface by which configs can be built from provider metadata type Builder interface { // CreateConfig creates a config from hints passed from providers - CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config + CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config } -// Builders is a struct of Builder list objects and a `kubernetesKeystoresRegistry` object, which -// holds Kubernetes keystores for known namespaces +// Builders is a struct of Builder list objects and a `keystoreProvider`, which +// has access to a keystores registry type Builders struct { - Builders []Builder - kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry + Builders []Builder + keystoreProvider keystore.KeystoreProvider } // BuilderConstructor is a func used to generate a Builder object @@ -98,8 +98,8 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { configs := []*common.Config{} var opts []ucfg.Option - if b.kubernetesKeystoresRegistry != nil { - k8sKeystore := b.kubernetesKeystoresRegistry.GetK8sKeystore(event) + if b.keystoreProvider != nil { + k8sKeystore := b.keystoreProvider.GetKeystore(event) if k8sKeystore != nil { opts = []ucfg.Option{ ucfg.Resolve(keystore.ResolverWrap(k8sKeystore)), @@ -107,7 +107,7 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { } } for _, builder := range b.Builders { - if config := builder.CreateConfig(event, opts); config != nil { + if config := builder.CreateConfig(event, opts...); config != nil { configs = append(configs, config...) } } @@ -115,14 +115,13 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { return configs } -// SetKubernetesKeystoresRegistry set the k8sKeystoresRegistry of the Builders object -func (b *Builders) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { - b.kubernetesKeystoresRegistry = k8sKeystoresRegistry -} - // NewBuilders instances the given list of builders. hintsCfg holds `hints` settings -// for simplified mode (single 'hints' builder) -func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) { +// for simplified mode (single 'hints' builder), `keystoreProvider` has access to keystore registry +func NewBuilders( + bConfigs []*common.Config, + hintsCfg *common.Config, + keystoreProvider keystore.KeystoreProvider, +) (Builders, error) { var builders Builders if hintsCfg.Enabled() { if len(bConfigs) > 0 { @@ -141,6 +140,6 @@ func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, } builders.Builders = append(builders.Builders, builder) } - + builders.keystoreProvider = keystoreProvider return builders, nil } diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 9df81aa63d98..154fe335e8c7 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -30,7 +30,7 @@ import ( type fakeBuilder struct{} -func (f *fakeBuilder) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { +func (f *fakeBuilder) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { return []*common.Config{common.NewConfig()} } @@ -63,7 +63,7 @@ func TestBuilderRegistry(t *testing.T) { // Try to create a config with fake builder and assert length // of configs returned is one - res := builder.CreateConfig(nil, nil) + res := builder.CreateConfig(nil) assert.Equal(t, len(res), 1) builders := Builders{} diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index f2600195b8f7..553b981177ec 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -77,7 +77,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(err) } - mapper, err := template.NewConfigMapper(config.Templates, keystore) + mapper, err := template.NewConfigMapper(config.Templates, keystore, nil) if err != nil { return nil, errWrap(err) } @@ -85,7 +85,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs or hints defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, nil) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index 9fcb64823bf8..5a8876a011ac 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -69,7 +69,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore Interfaces: config.Interfaces, } - mapper, err := template.NewConfigMapper(config.Templates, keystore) + mapper, err := template.NewConfigMapper(config.Templates, keystore, nil) if err != nil { return nil, errWrap(err) } @@ -77,7 +77,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(fmt.Errorf("no configs defined for autodiscover provider")) } - builders, err := autodiscover.NewBuilders(config.Builders, nil) + builders, err := autodiscover.NewBuilders(config.Builders, nil, nil) if err != nil { return nil, errWrap(err) } diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index 4f0898f4f6aa..0cbe2d14ef24 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -78,6 +78,6 @@ func newMockBuilder(_ *common.Config) (autodiscover.Builder, error) { return &mockBuilder{}, nil } -func (m *mockBuilder) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { +func (m *mockBuilder) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { return nil } diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index c62b8b3f2d8c..289c0cd098e4 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -30,7 +30,8 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/keystore" + k8sks "github.com/elastic/beats/v7/libbeat/k8skeystore" + ks "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -58,7 +59,7 @@ type Provider struct { } // AutodiscoverBuilder builds and returns an autodiscover provider -func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore keystore.Keystore) (autodiscover.Provider, error) { +func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore ks.Keystore) (autodiscover.Provider, error) { logger := logp.NewLogger("autodiscover") errWrap := func(err error) error { @@ -76,12 +77,14 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore return nil, errWrap(err) } - mapper, err := template.NewConfigMapper(config.Templates, keyStore) + k8sKeystoreProvider := k8sks.NewKubernetesKeystoresRegistry(logger, client) + + mapper, err := template.NewConfigMapper(config.Templates, keystore, k8sKeystoreProvider) if err != nil { return nil, errWrap(err) } - builders, err := autodiscover.NewBuilders(config.Builders, config.Hints) + builders, err := autodiscover.NewBuilders(config.Builders, config.Hints, k8sKeystoreProvider) if err != nil { return nil, errWrap(err) } @@ -91,10 +94,6 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keyStore return nil, errWrap(err) } - k8sKeystoresRegistry := keystore.NewKubernetesKeystoresRegistry(logger, client) - mapper.SetKubernetesKeystoresRegistry(k8sKeystoresRegistry) - builders.SetKubernetesKeystoresRegistry(k8sKeystoresRegistry) - p := &Provider{ config: config, bus: bus, diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index 9180f3291fa6..59fb67ada7da 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -240,7 +240,7 @@ func TestEmitEvent_Node(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil, nil) + mapper, err := template.NewConfigMapper(nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 2b14dbb3d42c..ce5375231675 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -633,7 +633,7 @@ func TestEmitEvent(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil, nil) + mapper, err := template.NewConfigMapper(nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index 241f3f2a986a..7ead61fc3a03 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -392,7 +392,7 @@ func TestEmitEvent_Service(t *testing.T) { for _, test := range tests { t.Run(test.Message, func(t *testing.T) { - mapper, err := template.NewConfigMapper(nil, nil) + mapper, err := template.NewConfigMapper(nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 002b09672c41..6fe25bc6d8c4 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -29,12 +29,12 @@ import ( // Mapper maps config templates with conditions in ConditionMaps, if a match happens on a discover event // the given template will be used as config. -// Mapper also includes the global Keystore object at `keystore` and `kubernetesKeystoresRegistry`, which -// holds Kubernetes keystores for known namespaces +// Mapper also includes the global Keystore object at `keystore` and `keystoreProvider`, which +// has access to a keystores registry type Mapper struct { - ConditionMaps []*ConditionMap - keystore keystore.Keystore - kubernetesKeystoresRegistry *keystore.KubernetesKeystoresRegistry + ConditionMaps []*ConditionMap + keystore keystore.Keystore + keystoreProvider keystore.KeystoreProvider } // ConditionMap maps a condition to the configs to use when it's triggered @@ -50,7 +50,11 @@ type MapperSettings []*struct { } // NewConfigMapper builds a template Mapper from given settings -func NewConfigMapper(configs MapperSettings, keystore keystore.Keystore) (mapper Mapper, err error) { +func NewConfigMapper( + configs MapperSettings, + keystore keystore.Keystore, + keystoreProvider keystore.KeystoreProvider, +) (mapper Mapper, err error) { for _, c := range configs { condMap := &ConditionMap{Configs: c.Configs} if c.ConditionConfig != nil { @@ -63,6 +67,7 @@ func NewConfigMapper(configs MapperSettings, keystore keystore.Keystore) (mapper } mapper.keystore = keystore + mapper.keystoreProvider = keystoreProvider return mapper, nil } @@ -78,11 +83,6 @@ func (e Event) GetValue(key string) (interface{}, error) { return val, nil } -// SetKubernetesKeystoresRegistry set the k8sKeystoresRegistry of the Mapper object -func (c *Mapper) SetKubernetesKeystoresRegistry(k8sKeystoresRegistry *keystore.KubernetesKeystoresRegistry) { - c.kubernetesKeystoresRegistry = k8sKeystoresRegistry -} - // GetConfig returns a matching Config if any, nil otherwise func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config @@ -90,8 +90,8 @@ func (c Mapper) GetConfig(event bus.Event) []*common.Config { if c.keystore != nil { opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(c.keystore))) } - if c.kubernetesKeystoresRegistry != nil { - k8sKeystore := c.kubernetesKeystoresRegistry.GetK8sKeystore(event) + if c.keystoreProvider != nil { + k8sKeystore := c.keystoreProvider.GetKeystore(event) if k8sKeystore != nil { opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(k8sKeystore))) } @@ -103,7 +103,7 @@ func (c Mapper) GetConfig(event bus.Event) []*common.Config { continue } - configs := ApplyConfigTemplate(event, mapping.Configs, opts) + configs := ApplyConfigTemplate(event, mapping.Configs, opts...) if configs != nil { result = append(result, configs...) } @@ -112,7 +112,7 @@ func (c Mapper) GetConfig(event bus.Event) []*common.Config { } // ApplyConfigTemplate takes a set of templated configs and applys information in an event map -func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options []ucfg.Option) []*common.Config { +func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options ...ucfg.Option) []*common.Config { var result []*common.Config // unpack input vars, err := ucfg.NewFrom(map[string]interface{}{ diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index f2bf33bf3dbb..f215f2e2899d 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -87,7 +87,7 @@ func TestConfigsMapping(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings, nil) + mapper, err := NewConfigMapper(mappings, nil, nil) if err != nil { t.Fatal(err) } @@ -141,7 +141,7 @@ func TestConfigsMappingKeystore(t *testing.T) { t.Fatal(err) } - mapper, err := NewConfigMapper(mappings, keystore) + mapper, err := NewConfigMapper(mappings, keystore, nil) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func TestNilConditionConfig(t *testing.T) { t.Fatal(err) } - _, err = NewConfigMapper(mappings, nil) + _, err = NewConfigMapper(mappings, nil, nil) assert.NoError(t, err) assert.Nil(t, mappings[0].ConditionConfig) } diff --git a/libbeat/keystore/kubernetes_keystore.go b/libbeat/k8skeystore/kubernetes_keystore.go similarity index 81% rename from libbeat/keystore/kubernetes_keystore.go rename to libbeat/k8skeystore/kubernetes_keystore.go index 5d4cab7c7a16..3863d4f3a1b4 100644 --- a/libbeat/keystore/kubernetes_keystore.go +++ b/libbeat/k8skeystore/kubernetes_keystore.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package keystore +package k8skeystore import ( "strings" @@ -25,10 +25,11 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/logp" ) -type KubernetesKeystores map[string]Keystore +type KubernetesKeystores map[string]keystore.Keystore // KubernetesKeystoresRegistry holds KubernetesKeystores for known namespaces. Once a Keystore for one k8s namespace // is initialized it will be reused every time it is needed. @@ -46,13 +47,13 @@ type KubernetesSecretsKeystore struct { } // Factoryk8s Create the right keystore with the configured options -func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (Keystore, error) { +func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) return keystore, err } // NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry -func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) *KubernetesKeystoresRegistry { +func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) keystore.KeystoreProvider { return &KubernetesKeystoresRegistry{ kubernetesKeystores: KubernetesKeystores{}, logger: logger, @@ -60,8 +61,8 @@ func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) * } } -// GetK8sKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. -func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore { +// GetKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. +func (kr *KubernetesKeystoresRegistry) GetKeystore(event bus.Event) keystore.Keystore { namespace := "" if val, ok := event["kubernetes"]; ok { kubernetesMeta := val.(common.MapStr) @@ -74,8 +75,7 @@ func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore } if namespace != "" { // either retrieve already stored keystore or create a new one for the namespace - storedKeystore := kr.lookupForKeystore(namespace) - if storedKeystore != nil { + if storedKeystore, ok := kr.kubernetesKeystores[namespace]; ok { return storedKeystore } k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger) @@ -86,15 +86,8 @@ func (kr *KubernetesKeystoresRegistry) GetK8sKeystore(event bus.Event) Keystore return nil } -func (kr *KubernetesKeystoresRegistry) lookupForKeystore(keystoreNamespace string) Keystore { - if keystore, ok := kr.kubernetesKeystores[keystoreNamespace]; ok { - return keystore - } - return nil -} - // NewKubernetesSecretsKeystore returns an new k8s Keystore -func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (Keystore, error) { +func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { keystore := KubernetesSecretsKeystore{ namespace: keystoreNamespace, client: ks8client, @@ -104,7 +97,7 @@ func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interf } // Retrieve return a SecureString instance that will contains both the key and the secret. -func (k *KubernetesSecretsKeystore) Retrieve(key string) (*SecureString, error) { +func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString, error) { // key = "kubernetes.somenamespace.somesecret.value" tokens := strings.Split(key, ".") if len(tokens) != 4 { @@ -113,32 +106,32 @@ func (k *KubernetesSecretsKeystore) Retrieve(key string) (*SecureString, error) key, "kubernetes.somenamespace.somesecret.value", ) - return nil, ErrKeyDoesntExists + return nil, keystore.ErrKeyDoesntExists } ns := tokens[1] secretName := tokens[2] secretVar := tokens[3] if ns != k.namespace { k.logger.Debugf("cannot access Kubernetes secrets from a different namespace than: %v", ns) - return nil, ErrKeyDoesntExists + return nil, keystore.ErrKeyDoesntExists } secretIntefrace := k.client.CoreV1().Secrets(ns) secrets, err := secretIntefrace.List(metav1.ListOptions{}) if err != nil { k.logger.Errorf("Could not retrieve secrets from k8s API: %v", err) - return nil, ErrKeyDoesntExists + return nil, keystore.ErrKeyDoesntExists } if len(secrets.Items) == 0 { k.logger.Debugf("no secrets found for namespace: %v", ns) - return nil, ErrKeyDoesntExists + return nil, keystore.ErrKeyDoesntExists } secret, err := secretIntefrace.Get(secretName, metav1.GetOptions{}) if err != nil { k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) - return nil, ErrKeyDoesntExists + return nil, keystore.ErrKeyDoesntExists } secretString := secret.Data[secretVar] - return NewSecureString(secretString), nil + return keystore.NewSecureString(secretString), nil } // GetConfig returns common.Config representation of the key / secret pair to be merged with other diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go index 340b83eb416c..e25bdc82b4e5 100644 --- a/libbeat/keystore/keystore.go +++ b/libbeat/keystore/keystore.go @@ -21,7 +21,8 @@ import ( "errors" "github.com/elastic/beats/v7/libbeat/common" - ucfg "github.com/elastic/go-ucfg" + "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/parse" ) @@ -73,6 +74,11 @@ type ListingKeystore interface { List() ([]string, error) } +// Provider for keystore +type KeystoreProvider interface { + GetKeystore(event bus.Event) Keystore +} + // ResolverWrap wrap a config resolver around an existing keystore. func ResolverWrap(keystore Keystore) func(string) (string, parse.Config, error) { return func(keyName string) (string, parse.Config, error) { diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index cf5927e98d74..b366789ba275 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -70,7 +70,7 @@ func NewMetricHints(cfg *common.Config) (autodiscover.Builder, error) { } // Create configs based on hints passed from providers -func (m *metricHints) CreateConfig(event bus.Event, options []ucfg.Option) []*common.Config { +func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { var config []*common.Config host, _ := event["host"].(string) if host == "" { @@ -95,7 +95,7 @@ func (m *metricHints) CreateConfig(event bus.Event, options []ucfg.Option) []*co } logp.Debug("hints.builder", "generated config %+v", configs) // Apply information in event to the template to generate the final config - return template.ApplyConfigTemplate(event, configs, options) + return template.ApplyConfigTemplate(event, configs, options...) } @@ -156,7 +156,7 @@ func (m *metricHints) CreateConfig(event bus.Event, options []ucfg.Option) []*co // Apply information in event to the template to generate the final config // This especially helps in a scenario where endpoints are configured as: // co.elastic.metrics/hosts= "${data.host}:9090" - return template.ApplyConfigTemplate(event, config, options) + return template.ApplyConfigTemplate(event, config, options...) } func (m *metricHints) getModule(hints common.MapStr) string { diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index f0488ecaa73c..a6dddb6b7a1a 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -302,7 +302,7 @@ func TestGenerateHints(t *testing.T) { Key: defaultConfig().Key, Registry: mockRegister, } - cfgs := m.CreateConfig(test.event, nil) + cfgs := m.CreateConfig(test.event) assert.Equal(t, len(cfgs), test.len) if len(cfgs) != 0 { @@ -375,7 +375,7 @@ func TestGenerateHintsDoesNotAccessGlobalKeystore(t *testing.T) { Key: defaultConfig().Key, Registry: mockRegister, } - cfgs := m.CreateConfig(test.event, nil) + cfgs := m.CreateConfig(test.event) assert.Equal(t, len(cfgs), test.len) if len(cfgs) != 0 { config := common.MapStr{} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 9374d60970ed..06b153626fe1 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -85,7 +85,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore // internalBuilder is mainly intended for testing via mocks and stubs. // it can be configured to use a fetcher that doesn't actually hit the AWS API. func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher, keystore keystore.Keystore) (*Provider, error) { - mapper, err := template.NewConfigMapper(config.Templates, keystore) + mapper, err := template.NewConfigMapper(config.Templates, keystore, nil) if err != nil { return nil, err } diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index f28fbf7f4cfa..54021a8db883 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -92,7 +92,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore // internalBuilder is mainly intended for testing via mocks and stubs. // it can be configured to use a fetcher that doesn't actually hit the AWS API. func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher, keystore keystore.Keystore) (*Provider, error) { - mapper, err := template.NewConfigMapper(config.Templates, keystore) + mapper, err := template.NewConfigMapper(config.Templates, keystore, nil) if err != nil { return nil, err } From 2e139c682ece7dcc095d94b12d02dfd5a935392f Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 26 May 2020 13:14:23 +0300 Subject: [PATCH 14/18] Add provider test Signed-off-by: chrismark --- libbeat/autodiscover/template/config_test.go | 72 ++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index f215f2e2899d..1ef0175c4843 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -151,6 +151,78 @@ func TestConfigsMappingKeystore(t *testing.T) { } } +func TestConfigsMappingKeystoreProvider(t *testing.T) { + secret := "mapping_provider_secret" + //expected config + config, _ := common.NewConfigFrom(map[string]interface{}{ + "correct": "config", + "password": secret, + }) + + path := getTemporaryKeystoreFile() + defer os.Remove(path) + // store the secret + keystore := createAnExistingKeystore(path, secret) + + tests := []struct { + mapping string + event bus.Event + expected []*common.Config + }{ + // Match config + { + mapping: ` +- condition.equals: + foo: 3 + config: + - correct: config + password: "${PASSWORD}"`, + event: bus.Event{ + "foo": 3, + }, + expected: []*common.Config{config}, + }, + } + + keystoreProvider := newMockKeystoreProvider(secret) + for _, test := range tests { + var mappings MapperSettings + config, err := common.NewConfigWithYAML([]byte(test.mapping), "") + if err != nil { + t.Fatal(err) + } + + if err := config.Unpack(&mappings); err != nil { + t.Fatal(err) + } + + mapper, err := NewConfigMapper(mappings, keystore, keystoreProvider) + if err != nil { + t.Fatal(err) + } + + res := mapper.GetConfig(test.event) + assert.Equal(t, test.expected, res) + } +} + +type mockKeystore struct { + secret string +} + +func newMockKeystoreProvider(secret string) keystore.KeystoreProvider { + return &mockKeystore{secret} +} + +// GetKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. +func (kr *mockKeystore) GetKeystore(event bus.Event) keystore.Keystore { + path := getTemporaryKeystoreFile() + defer os.Remove(path) + // store the secret + keystore := createAnExistingKeystore(path, kr.secret) + return keystore +} + func TestNilConditionConfig(t *testing.T) { var mappings MapperSettings data := ` From e02314d2e72a221517c1f893a0d97ee4c405ed26 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 26 May 2020 14:49:44 +0300 Subject: [PATCH 15/18] k8s Keystore tests Signed-off-by: chrismark --- .../k8skeystore/kubernetes_keystore_test.go | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 libbeat/k8skeystore/kubernetes_keystore_test.go diff --git a/libbeat/k8skeystore/kubernetes_keystore_test.go b/libbeat/k8skeystore/kubernetes_keystore_test.go new file mode 100644 index 000000000000..60041bc0a080 --- /dev/null +++ b/libbeat/k8skeystore/kubernetes_keystore_test.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package k8skeystore + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/bus" +) + +func TestGetKeystore(t *testing.T) { + kRegistry := NewKubernetesKeystoresRegistry(nil, nil) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": common.MapStr{"namespace": "my_namespace"}}) + k2 := kRegistry.GetKeystore(bus.Event{"kubernetes": common.MapStr{"namespace": "my_namespace"}}) + assert.Equal(t, k1, k2) + k3 := kRegistry.GetKeystore(bus.Event{"kubernetes": common.MapStr{"namespace": "my_namespace_2"}}) + assert.NotEqual(t, k2, k3) +} + +// TODO: upgrade client dependency and use fake client to test retrieve +//func TestGetKeystoreAndRetrieve(t *testing.T) { +// client := k8sfake.NewSimpleClientset() +// ns := "test_namespace" +// pass := "testing_passpass" +// secret := &v1.Secret{ +// TypeMeta: metav1.TypeMeta{ +// Kind: "Secret", +// APIVersion: "apps/v1beta1", +// }, +// ObjectMeta: metav1.ObjectMeta{ +// Name: "testing_secret", +// Namespace: ns, +// }, +// Data: map[string][]byte{ +// "secret_value": []byte(pass), +// }, +// } +// client.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{}) +// +// kRegistry := NewKubernetesKeystoresRegistry(nil, nil) +// k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": common.MapStr{"namespace": ns}}) +// key := "kubernetes.test_namespace.testing_secret.secret_value" +// secretVal, err := k1.Retrieve(key) +// if err != nil { +// t.Fatalf("could not retrive k8s secret", err) +// } +// assert.Equal(t, pass, secretVal) +//} From 55070efce8708ab51f5dbab7bb59a2dcc16afb19 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 26 May 2020 15:50:25 +0300 Subject: [PATCH 16/18] Review changes Signed-off-by: chrismark --- .../autodiscover/builder/hints/monitors_test.go | 2 +- libbeat/autodiscover/builder.go | 12 ++++++------ libbeat/autodiscover/builder_test.go | 2 +- .../autodiscover/providers/kubernetes/kubernetes.go | 8 ++++---- libbeat/autodiscover/template/config.go | 12 +++++++----- libbeat/autodiscover/template/config_test.go | 2 +- .../kubernetes}/k8skeystore/kubernetes_keystore.go | 10 +++++----- .../k8skeystore/kubernetes_keystore_test.go | 4 ++-- libbeat/keystore/keystore.go | 2 +- 9 files changed, 28 insertions(+), 26 deletions(-) rename libbeat/{ => common/kubernetes}/k8skeystore/kubernetes_keystore.go (95%) rename libbeat/{ => common/kubernetes}/k8skeystore/kubernetes_keystore_test.go (95%) diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index e5ed7ec44910..82c4dc6262c8 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -203,7 +203,7 @@ func TestGenerateHints(t *testing.T) { config: defaultConfig(), logger: logp.NewLogger("hints.builder"), } - cfgs := m.CreateConfig(test.event, nil) + cfgs := m.CreateConfig(test.event) assert.Equal(t, len(cfgs), test.len, test.message) if len(cfgs) != 0 { diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index 0f57b187ea05..ba091b46c4e6 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -35,11 +35,11 @@ type Builder interface { CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config } -// Builders is a struct of Builder list objects and a `keystoreProvider`, which +// builders is a struct of Builder list objects and a `keystoreProvider`, which // has access to a keystores registry type Builders struct { - Builders []Builder - keystoreProvider keystore.KeystoreProvider + builders []Builder + keystoreProvider keystore.Provider } // BuilderConstructor is a func used to generate a Builder object @@ -106,7 +106,7 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { } } } - for _, builder := range b.Builders { + for _, builder := range b.builders { if config := builder.CreateConfig(event, opts...); config != nil { configs = append(configs, config...) } @@ -120,7 +120,7 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config { func NewBuilders( bConfigs []*common.Config, hintsCfg *common.Config, - keystoreProvider keystore.KeystoreProvider, + keystoreProvider keystore.Provider, ) (Builders, error) { var builders Builders if hintsCfg.Enabled() { @@ -138,7 +138,7 @@ func NewBuilders( if err != nil { return Builders{}, err } - builders.Builders = append(builders.Builders, builder) + builders.builders = append(builders.builders, builder) } builders.keystoreProvider = keystoreProvider return builders, nil diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 154fe335e8c7..c5e08a77e395 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -67,7 +67,7 @@ func TestBuilderRegistry(t *testing.T) { assert.Equal(t, len(res), 1) builders := Builders{} - builders.Builders = append(builders.Builders, builder) + builders.builders = append(builders.builders, builder) // Try using builders object for the same as above and expect // the same result diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 289c0cd098e4..e1a2cb02ee0f 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -30,8 +30,8 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/kubernetes" - k8sks "github.com/elastic/beats/v7/libbeat/k8skeystore" - ks "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/common/kubernetes/k8skeystore" + "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -59,7 +59,7 @@ type Provider struct { } // AutodiscoverBuilder builds and returns an autodiscover provider -func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore ks.Keystore) (autodiscover.Provider, error) { +func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore keystore.Keystore) (autodiscover.Provider, error) { logger := logp.NewLogger("autodiscover") errWrap := func(err error) error { @@ -77,7 +77,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore return nil, errWrap(err) } - k8sKeystoreProvider := k8sks.NewKubernetesKeystoresRegistry(logger, client) + k8sKeystoreProvider := k8skeystore.NewKubernetesKeystoresRegistry(logger, client) mapper, err := template.NewConfigMapper(config.Templates, keystore, k8sKeystoreProvider) if err != nil { diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 6fe25bc6d8c4..a34cec104447 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -34,7 +34,7 @@ import ( type Mapper struct { ConditionMaps []*ConditionMap keystore keystore.Keystore - keystoreProvider keystore.KeystoreProvider + keystoreProvider keystore.Provider } // ConditionMap maps a condition to the configs to use when it's triggered @@ -53,7 +53,7 @@ type MapperSettings []*struct { func NewConfigMapper( configs MapperSettings, keystore keystore.Keystore, - keystoreProvider keystore.KeystoreProvider, + keystoreProvider keystore.Provider, ) (mapper Mapper, err error) { for _, c := range configs { condMap := &ConditionMap{Configs: c.Configs} @@ -87,15 +87,17 @@ func (e Event) GetValue(key string) (interface{}, error) { func (c Mapper) GetConfig(event bus.Event) []*common.Config { var result []*common.Config opts := []ucfg.Option{} - if c.keystore != nil { - opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(c.keystore))) - } + // add k8s keystore in options list with higher priority if c.keystoreProvider != nil { k8sKeystore := c.keystoreProvider.GetKeystore(event) if k8sKeystore != nil { opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(k8sKeystore))) } } + // add local keystore in options list with lower priority + if c.keystore != nil { + opts = append(opts, ucfg.Resolve(keystore.ResolverWrap(c.keystore))) + } for _, mapping := range c.ConditionMaps { // An empty condition matches everything conditionOk := mapping.Condition == nil || mapping.Condition.Check(Event(event)) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index 1ef0175c4843..933c4fbb1c85 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -210,7 +210,7 @@ type mockKeystore struct { secret string } -func newMockKeystoreProvider(secret string) keystore.KeystoreProvider { +func newMockKeystoreProvider(secret string) keystore.Provider { return &mockKeystore{secret} } diff --git a/libbeat/k8skeystore/kubernetes_keystore.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go similarity index 95% rename from libbeat/k8skeystore/kubernetes_keystore.go rename to libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go index 3863d4f3a1b4..3ac7998579a8 100644 --- a/libbeat/k8skeystore/kubernetes_keystore.go +++ b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go @@ -23,10 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/libbeat/keystore" + "github.com/elastic/beats/libbeat/logp" ) type KubernetesKeystores map[string]keystore.Keystore @@ -53,7 +53,7 @@ func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp. } // NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry -func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) keystore.KeystoreProvider { +func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) keystore.Provider { return &KubernetesKeystoresRegistry{ kubernetesKeystores: KubernetesKeystores{}, logger: logger, diff --git a/libbeat/k8skeystore/kubernetes_keystore_test.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go similarity index 95% rename from libbeat/k8skeystore/kubernetes_keystore_test.go rename to libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go index 60041bc0a080..965060727ca7 100644 --- a/libbeat/k8skeystore/kubernetes_keystore_test.go +++ b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/bus" ) func TestGetKeystore(t *testing.T) { diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go index e25bdc82b4e5..2dbf7ff144f8 100644 --- a/libbeat/keystore/keystore.go +++ b/libbeat/keystore/keystore.go @@ -75,7 +75,7 @@ type ListingKeystore interface { } // Provider for keystore -type KeystoreProvider interface { +type Provider interface { GetKeystore(event bus.Event) Keystore } From 865957067bdf652a0f673f2467cdcf87bf0e83a1 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 26 May 2020 15:52:58 +0300 Subject: [PATCH 17/18] fix imports Signed-off-by: chrismark --- .../common/kubernetes/k8skeystore/kubernetes_keystore.go | 8 ++++---- .../kubernetes/k8skeystore/kubernetes_keystore_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go index 3ac7998579a8..a4a126158985 100644 --- a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go +++ b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go @@ -23,10 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/bus" - "github.com/elastic/beats/libbeat/keystore" - "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/logp" ) type KubernetesKeystores map[string]keystore.Keystore diff --git a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go index 965060727ca7..60041bc0a080 100644 --- a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go +++ b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/bus" ) func TestGetKeystore(t *testing.T) { From 5a7e385af2b23c4b6518f01642b96b29097525be Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 27 May 2020 11:30:36 +0300 Subject: [PATCH 18/18] Improve logging message and key check Signed-off-by: chrismark --- .../common/kubernetes/k8skeystore/kubernetes_keystore.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go index a4a126158985..23a2c3fcf6cc 100644 --- a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go +++ b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go @@ -112,7 +112,7 @@ func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString secretName := tokens[2] secretVar := tokens[3] if ns != k.namespace { - k.logger.Debugf("cannot access Kubernetes secrets from a different namespace than: %v", ns) + k.logger.Debugf("cannot access Kubernetes secrets from a different namespace (%v) than: %v", ns, k.namespace) return nil, keystore.ErrKeyDoesntExists } secretIntefrace := k.client.CoreV1().Secrets(ns) @@ -130,6 +130,10 @@ func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) return nil, keystore.ErrKeyDoesntExists } + if _, ok := secret.Data[secretVar]; !ok { + k.logger.Errorf("Could not retrieve value %v for secret %v", secretVar, secretName) + return nil, keystore.ErrKeyDoesntExists + } secretString := secret.Data[secretVar] return keystore.NewSecureString(secretString), nil }