From c65acd4ba0980a2378f81f678bad6bff6eaac5dd Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Apr 2020 15:09:57 -0700 Subject: [PATCH 1/7] Deep merge event fields and metadata maps --- libbeat/common/jsontransform/jsonhelper.go | 24 ++-- .../common/jsontransform/jsonhelper_test.go | 136 ++++++++++++++++++ 2 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 libbeat/common/jsontransform/jsonhelper_test.go diff --git a/libbeat/common/jsontransform/jsonhelper.go b/libbeat/common/jsontransform/jsonhelper.go index 1490bcff170..8efe76be0e7 100644 --- a/libbeat/common/jsontransform/jsonhelper.go +++ b/libbeat/common/jsontransform/jsonhelper.go @@ -29,11 +29,13 @@ import ( // WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option and the addErrKey func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys bool, addErrKey bool) { if !overwriteKeys { - for k, v := range keys { - if _, exists := event.Fields[k]; !exists && k != "@timestamp" && k != "@metadata" { - event.Fields[k] = v - } - } + // @timestamp and @metadata fields are root-level fields. We remove them so they + // don't become part of event.Fields. + delete(keys, "@timestamp") + delete(keys, "@metadata") + + // Then, perform deep update without overwriting + event.Fields.DeepUpdateNoOverwrite(keys) return } @@ -64,7 +66,7 @@ func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys } case map[string]interface{}: - event.Meta.Update(common.MapStr(m)) + event.Meta.DeepUpdate(common.MapStr(m)) default: event.SetErrorWithOption(createJSONError("failed to update @metadata"), addErrKey) @@ -83,11 +85,15 @@ func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys continue } event.Fields[k] = vstr - - default: - event.Fields[k] = v } } + + // We have accounted for @timestamp, @metadata, type above. So let's remove these keys and + // deep update the event with the rest of the keys + delete(keys, "@timestamp") + delete(keys, "@metadata") + delete(keys, "type") + event.Fields.DeepUpdate(keys) } func createJSONError(message string) common.MapStr { diff --git a/libbeat/common/jsontransform/jsonhelper_test.go b/libbeat/common/jsontransform/jsonhelper_test.go new file mode 100644 index 00000000000..bc3433badb4 --- /dev/null +++ b/libbeat/common/jsontransform/jsonhelper_test.go @@ -0,0 +1,136 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package jsontransform + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestWriteJSONKeys(t *testing.T) { + now := time.Now() + now = now.Round(time.Second) + + eventTimestamp := time.Date(2020, 01, 01, 01, 01, 00, 0, time.UTC) + eventMetadata := common.MapStr{ + "foo": "bar", + "baz": common.MapStr{ + "qux": 17, + }, + } + eventFields := common.MapStr{ + "top_a": 23, + "top_b": common.MapStr{ + "inner_c": "see", + "inner_d": "dee", + }, + } + + tests := map[string]struct { + keys map[string]interface{} + overwriteKeys bool + expectedMetadata common.MapStr + expectedTimestamp time.Time + expectedFields common.MapStr + }{ + "overwrite_true": { + overwriteKeys: true, + keys: map[string]interface{}{ + "@metadata": map[string]interface{}{ + "foo": "NEW_bar", + "baz": map[string]interface{}{ + "qux": "NEW_qux", + "durrr": "COMPLETELY_NEW", + }, + }, + "@timestamp": now.Format(time.RFC3339), + "top_b": map[string]interface{}{ + "inner_d": "NEW_dee", + "inner_e": "COMPLETELY_NEW_e", + }, + "top_c": "COMPLETELY_NEW_c", + }, + expectedMetadata: common.MapStr{ + "foo": "NEW_bar", + "baz": common.MapStr{ + "qux": "NEW_qux", + "durrr": "COMPLETELY_NEW", + }, + }, + expectedTimestamp: now, + expectedFields: common.MapStr{ + "top_a": 23, + "top_b": common.MapStr{ + "inner_c": "see", + "inner_d": "NEW_dee", + "inner_e": "COMPLETELY_NEW_e", + }, + "top_c": "COMPLETELY_NEW_c", + }, + }, + "overwrite_false": { + overwriteKeys: false, + keys: map[string]interface{}{ + "@metadata": map[string]interface{}{ + "foo": "NEW_bar", + "baz": map[string]interface{}{ + "qux": "NEW_qux", + "durrr": "COMPLETELY_NEW", + }, + }, + "@timestamp": now.Format(time.RFC3339), + "top_b": map[string]interface{}{ + "inner_d": "NEW_dee", + "inner_e": "COMPLETELY_NEW_e", + }, + "top_c": "COMPLETELY_NEW_c", + }, + expectedMetadata: eventMetadata.Clone(), + expectedTimestamp: eventTimestamp, + expectedFields: common.MapStr{ + "top_a": 23, + "top_b": common.MapStr{ + "inner_c": "see", + "inner_d": "dee", + "inner_e": "COMPLETELY_NEW_e", + }, + "top_c": "COMPLETELY_NEW_c", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + event := &beat.Event{ + Timestamp: eventTimestamp, + Meta: eventMetadata.Clone(), + Fields: eventFields.Clone(), + } + + WriteJSONKeys(event, test.keys, test.overwriteKeys, false) + require.Equal(t, test.expectedMetadata, event.Meta) + require.Equal(t, test.expectedTimestamp, event.Timestamp) + require.Equal(t, test.expectedFields, event.Fields) + }) + } +} From 4137070b0831a49ee3970e661be9ddafd940608e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Apr 2020 15:25:16 -0700 Subject: [PATCH 2/7] Add CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b3eaf0a08dc..f474fabdecb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,6 +83,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030] - Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143] - Gives monitoring reporter hosts, if configured, total precedence over corresponding output hosts. {issue}17937[17937] {pull}17991[17991] +- Arbitrary fields and metadata maps are now deep merged into event. {pull}17958[17958] *Auditbeat* From 1b390a27b58ddb96ae8a478741966d8ce852cb3e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 24 Apr 2020 06:07:06 -0700 Subject: [PATCH 3/7] Use loop to remove keys; extract into function --- libbeat/common/jsontransform/jsonhelper.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/libbeat/common/jsontransform/jsonhelper.go b/libbeat/common/jsontransform/jsonhelper.go index 8efe76be0e7..164e1e9e1f4 100644 --- a/libbeat/common/jsontransform/jsonhelper.go +++ b/libbeat/common/jsontransform/jsonhelper.go @@ -30,9 +30,8 @@ import ( func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys bool, addErrKey bool) { if !overwriteKeys { // @timestamp and @metadata fields are root-level fields. We remove them so they - // don't become part of event.Fields. - delete(keys, "@timestamp") - delete(keys, "@metadata") + // don't become part of event.Fields. + removeKeys(keys, "@timestamp", "@metadata") // Then, perform deep update without overwriting event.Fields.DeepUpdateNoOverwrite(keys) @@ -89,13 +88,17 @@ func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys } // We have accounted for @timestamp, @metadata, type above. So let's remove these keys and - // deep update the event with the rest of the keys - delete(keys, "@timestamp") - delete(keys, "@metadata") - delete(keys, "type") + // deep update the event with the rest of the keys. + removeKeys(keys, "@timestamp", "@metadata", "type") event.Fields.DeepUpdate(keys) } func createJSONError(message string) common.MapStr { return common.MapStr{"message": message, "type": "json"} } + +func removeKeys(keys map[string]interface{}, names ...string) { + for _, name := range names { + delete(keys, name) + } +} From 9ca4d4888c3ef495197d55dabfb2b84bddcb827f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 24 Apr 2020 06:29:27 -0700 Subject: [PATCH 4/7] Relocating CHANGELOG entry --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f474fabdecb..f2203e4a696 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191] - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000] - Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153] +- When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958] *Auditbeat* @@ -295,6 +296,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added an input option `publisher_pipeline.disable_host` to disable `host.name` from being added to events by default. {pull}18159[18159] - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065] +- When using the `json.*` setting available on some inputs, decoded fields are now deep-merged into existing event. {pull}17958[17958] *Heartbeat* From f0a7aa6bbf5f6732b79bdf60bbf0cba4e831eeee Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Apr 2020 07:57:12 -0700 Subject: [PATCH 5/7] Rewording and moving to bugfix section per review feedback --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f2203e4a696..df0f6473d56 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -84,6 +84,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143] - Gives monitoring reporter hosts, if configured, total precedence over corresponding output hosts. {issue}17937[17937] {pull}17991[17991] - Arbitrary fields and metadata maps are now deep merged into event. {pull}17958[17958] +- Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] *Auditbeat* From 38dd5e5b1b6d22836350b70c6fe4f03a10c3bb71 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Apr 2020 05:53:29 -0700 Subject: [PATCH 6/7] Adapting other CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index df0f6473d56..b59dbb74c24 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -298,6 +298,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d from being added to events by default. {pull}18159[18159] - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065] - When using the `json.*` setting available on some inputs, decoded fields are now deep-merged into existing event. {pull}17958[17958] +- Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] *Heartbeat* From 2a9821b941c1d6963a75f54dce2a090bad3e6ac7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 May 2020 14:23:54 -0700 Subject: [PATCH 7/7] Fix comparison in test --- libbeat/common/jsontransform/jsonhelper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/jsontransform/jsonhelper_test.go b/libbeat/common/jsontransform/jsonhelper_test.go index bc3433badb4..d7679579be1 100644 --- a/libbeat/common/jsontransform/jsonhelper_test.go +++ b/libbeat/common/jsontransform/jsonhelper_test.go @@ -129,7 +129,7 @@ func TestWriteJSONKeys(t *testing.T) { WriteJSONKeys(event, test.keys, test.overwriteKeys, false) require.Equal(t, test.expectedMetadata, event.Meta) - require.Equal(t, test.expectedTimestamp, event.Timestamp) + require.Equal(t, test.expectedTimestamp.UnixNano(), event.Timestamp.UnixNano()) require.Equal(t, test.expectedFields, event.Fields) }) }