From db3d0acae0eba61eedbda14dae28d96302199ae6 Mon Sep 17 00:00:00 2001
From: Michael Katsoulis <michaelkatsoulis88@gmail.com>
Date: Tue, 24 Oct 2023 11:23:25 +0300
Subject: [PATCH] Use Merge with AppendValues option when merging default
 config with hints generated config (#36857)

* Use Merge with AppendValues option when merging default config with hints generated config

* Update filebeat/autodiscover/builder/hints/logs.go

Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co>

* Update changelog

---------

Co-authored-by: Andrew Gizas <andreas.gkizas@elastic.co>
---
 CHANGELOG.next.asciidoc                       |  1 +
 filebeat/autodiscover/builder/hints/logs.go   |  3 +-
 .../autodiscover/builder/hints/logs_test.go   | 73 +++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 7fbe4c0e480d..a8b019f0863b 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -112,6 +112,7 @@ is collected by it.
 - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
 - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
 - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772]
+- Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857]
 
 *Heartbeat*
 
diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
index 72d8e4a63dd2..52f8c91672a0 100644
--- a/filebeat/autodiscover/builder/hints/logs.go
+++ b/filebeat/autodiscover/builder/hints/logs.go
@@ -139,7 +139,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
 			kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log)
 		}
 		// Merge config template with the configs from the annotations
-		if err := config.Merge(tempCfg); err != nil {
+		// AppendValues option is used to append arrays from annotations to existing arrays while merging
+		if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil {
 			logp.Debug("hints.builder", "config merge failed with error: %v", err)
 			continue
 		}
diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go
index a8bba01d52bd..cd2b72367712 100644
--- a/filebeat/autodiscover/builder/hints/logs_test.go
+++ b/filebeat/autodiscover/builder/hints/logs_test.go
@@ -43,6 +43,24 @@ func TestGenerateHints(t *testing.T) {
 		},
 	})
 
+	customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{
+		"default_config": map[string]interface{}{
+			"type": "container",
+			"paths": []string{
+				"/var/lib/docker/containers/${data.container.id}/*-json.log",
+			},
+			"close_timeout": "true",
+			"processors": []interface{}{
+				map[string]interface{}{
+					"add_tags": map[string]interface{}{
+						"tags":   []string{"web"},
+						"target": "environment",
+					},
+				},
+			},
+		},
+	})
+
 	defaultCfg := conf.NewConfig()
 
 	defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{
@@ -389,6 +407,61 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			msg:    "Processors in hints must be appended in the processors of the default config",
+			config: customProcessorCfg,
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"kubernetes": mapstr.M{
+					"container": mapstr.M{
+						"name": "foobar",
+						"id":   "abc",
+					},
+				},
+				"container": mapstr.M{
+					"name": "foobar",
+					"id":   "abc",
+				},
+				"hints": mapstr.M{
+					"logs": mapstr.M{
+						"processors": mapstr.M{
+							"1": mapstr.M{
+								"dissect": mapstr.M{
+									"tokenizer": "%{key1} %{key2}",
+								},
+							},
+							"drop_event": mapstr.M{},
+						},
+					},
+				},
+			},
+			len: 1,
+			result: []mapstr.M{
+				{
+					"type": "container",
+					"paths": []interface{}{
+						"/var/lib/docker/containers/abc/*-json.log",
+					},
+					"close_timeout": "true",
+					"processors": []interface{}{
+						map[string]interface{}{
+							"add_tags": map[string]interface{}{
+								"tags":   []interface{}{"web"},
+								"target": "environment",
+							},
+						},
+						map[string]interface{}{
+							"dissect": map[string]interface{}{
+								"tokenizer": "%{key1} %{key2}",
+							},
+						},
+						map[string]interface{}{
+							"drop_event": nil,
+						},
+					},
+				},
+			},
+		},
 		{
 			msg:    "Hint with module should attach input to its filesets",
 			config: customCfg,