Use Merge with AppendValues option when merging default config with h…

…ints generated config (elastic#36857) * Use Merge with AppendValues option when merging default config with hints generated config * Update filebeat/autodiscover/builder/hints/logs.go Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co> * Update changelog --------- Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co>
Scholar-Li · Feb 5, 2024 · db3d0ac · db3d0ac
1 parent 859fa27
commit db3d0ac
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -112,6 +112,7 @@ is collected by it.
 - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
 - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
 - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772]
+- Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857]
 
 *Heartbeat*
 

diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
@@ -139,7 +139,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
  kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log)
  }
  // Merge config template with the configs from the annotations
- if err := config.Merge(tempCfg); err != nil {
+ // AppendValues option is used to append arrays from annotations to existing arrays while merging
+ if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil {
  logp.Debug("hints.builder", "config merge failed with error: %v", err)
  continue
  }

diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go
@@ -43,6 +43,24 @@ func TestGenerateHints(t *testing.T) {
  },
  })
 
+ customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{
+ "default_config": map[string]interface{}{
+ "type": "container",
+ "paths": []string{
+ "/var/lib/docker/containers/${data.container.id}/*-json.log",
+ },
+ "close_timeout": "true",
+ "processors": []interface{}{
+ map[string]interface{}{
+ "add_tags": map[string]interface{}{
+ "tags": []string{"web"},
+ "target": "environment",
+ },
+ },
+ },
+ },
+ })
+
  defaultCfg := conf.NewConfig()
 
  defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{
@@ -389,6 +407,61 @@ func TestGenerateHints(t *testing.T) {
  },
  },
  },
+ {
+ msg: "Processors in hints must be appended in the processors of the default config",
+ config: customProcessorCfg,
+ event: bus.Event{
+ "host": "1.2.3.4",
+ "kubernetes": mapstr.M{
+ "container": mapstr.M{
+ "name": "foobar",
+ "id": "abc",
+ },
+ },
+ "container": mapstr.M{
+ "name": "foobar",
+ "id": "abc",
+ },
+ "hints": mapstr.M{
+ "logs": mapstr.M{
+ "processors": mapstr.M{
+ "1": mapstr.M{
+ "dissect": mapstr.M{
+ "tokenizer": "%{key1} %{key2}",
+ },
+ },
+ "drop_event": mapstr.M{},
+ },
+ },
+ },
+ },
+ len: 1,
+ result: []mapstr.M{
+ {
+ "type": "container",
+ "paths": []interface{}{
+ "/var/lib/docker/containers/abc/*-json.log",
+ },
+ "close_timeout": "true",
+ "processors": []interface{}{
+ map[string]interface{}{
+ "add_tags": map[string]interface{}{
+ "tags": []interface{}{"web"},
+ "target": "environment",
+ },
+ },
+ map[string]interface{}{
+ "dissect": map[string]interface{}{
+ "tokenizer": "%{key1} %{key2}",
+ },
+ },
+ map[string]interface{}{
+ "drop_event": nil,
+ },
+ },
+ },
+ },
+ },
  {
  msg: "Hint with module should attach input to its filesets",
  config: customCfg,