diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7fbe4c0e480d..a8b019f0863b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -112,6 +112,7 @@ is collected by it. - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557] - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395] - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772] +- Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857] *Heartbeat* diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 72d8e4a63dd2..52f8c91672a0 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -139,7 +139,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log) } // Merge config template with the configs from the annotations - if err := config.Merge(tempCfg); err != nil { + // AppendValues option is used to append arrays from annotations to existing arrays while merging + if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil { logp.Debug("hints.builder", "config merge failed with error: %v", err) continue } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index a8bba01d52bd..cd2b72367712 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -43,6 +43,24 @@ func TestGenerateHints(t *testing.T) { }, }) + customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{ + "default_config": map[string]interface{}{ + "type": "container", + "paths": []string{ + "/var/lib/docker/containers/${data.container.id}/*-json.log", + }, + "close_timeout": "true", + "processors": []interface{}{ + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []string{"web"}, + "target": "environment", + }, + }, + }, + }, + }) + defaultCfg := conf.NewConfig() defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{ @@ -389,6 +407,61 @@ func TestGenerateHints(t *testing.T) { }, }, }, + { + msg: "Processors in hints must be appended in the processors of the default config", + config: customProcessorCfg, + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": mapstr.M{ + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + }, + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + "hints": mapstr.M{ + "logs": mapstr.M{ + "processors": mapstr.M{ + "1": mapstr.M{ + "dissect": mapstr.M{ + "tokenizer": "%{key1} %{key2}", + }, + }, + "drop_event": mapstr.M{}, + }, + }, + }, + }, + len: 1, + result: []mapstr.M{ + { + "type": "container", + "paths": []interface{}{ + "/var/lib/docker/containers/abc/*-json.log", + }, + "close_timeout": "true", + "processors": []interface{}{ + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []interface{}{"web"}, + "target": "environment", + }, + }, + map[string]interface{}{ + "dissect": map[string]interface{}{ + "tokenizer": "%{key1} %{key2}", + }, + }, + map[string]interface{}{ + "drop_event": nil, + }, + }, + }, + }, + }, { msg: "Hint with module should attach input to its filesets", config: customCfg,