From 4ea0f91ef3fd51c167a1e11432edc30656062013 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 18 Feb 2020 20:23:47 +0100 Subject: [PATCH] Enable netinfo in add metadata processors (#16077) Enable netinfo in add_host_metadata and add_observer_metadata processors by default. This network information metadata is used in observability solutions. Co-authored-by: Dan Roscigno --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 3 +-- filebeat/filebeat.reference.yml | 3 +-- heartbeat/heartbeat.reference.yml | 3 +-- journalbeat/journalbeat.reference.yml | 3 +-- libbeat/_meta/config.reference.yml.tmpl | 3 +-- .../add_host_metadata_test.go | 20 +++++++++---------- .../processors/add_host_metadata/config.go | 2 +- .../docs/add_host_metadata.asciidoc | 3 +-- .../add_observer_metadata_test.go | 20 +++++++++---------- .../add_observer_metadata/config.go | 2 +- .../docs/add_observer_metadata.asciidoc | 3 +-- metricbeat/metricbeat.reference.yml | 3 +-- packetbeat/packetbeat.reference.yml | 3 +-- winlogbeat/winlogbeat.reference.yml | 3 +-- x-pack/auditbeat/auditbeat.reference.yml | 3 +-- x-pack/filebeat/filebeat.reference.yml | 3 +-- .../functionbeat/functionbeat.reference.yml | 3 +-- x-pack/metricbeat/metricbeat.reference.yml | 3 +-- x-pack/winlogbeat/winlogbeat.reference.yml | 3 +-- 20 files changed, 38 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0be601ded77..960154d5897 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - Add document_id setting to decode_json_fields processor. {pull}15859[15859] +- Include network information by default on add_host_metadata and add_observer_metadata. {issue}15347[15347] {pull}16077[16077] - Add `aws_ec2` provider for autodiscover. {issue}12518[12518] {pull}14823[14823] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 2fe13ec5e5d..b215921da97 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -315,8 +315,7 @@ auditbeat.modules: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index bc65b50bd2a..23fbfef0cd0 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1020,8 +1020,7 @@ filebeat.inputs: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 60941544931..721ca902364 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -459,8 +459,7 @@ heartbeat.scheduler: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 949df01a29a..6c9ab897bad 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -253,8 +253,7 @@ setup.template.settings: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index a949cd43327..17714de707d 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -196,8 +196,7 @@ # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 57f4358b65d..1ef8035f625 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -64,21 +64,21 @@ func TestConfigDefault(t *testing.T) { assert.NotNil(t, v) v, err = newEvent.GetValue("host.ip") - assert.Error(t, err) - assert.Nil(t, v) + assert.NoError(t, err) + assert.NotNil(t, v) v, err = newEvent.GetValue("host.mac") - assert.Error(t, err) - assert.Nil(t, v) + assert.NoError(t, err) + assert.NotNil(t, v) } -func TestConfigNetInfoEnabled(t *testing.T) { +func TestConfigNetInfoDisabled(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{}, Timestamp: time.Now(), } testConfig, err := common.NewConfigFrom(map[string]interface{}{ - "netinfo.enabled": true, + "netinfo.enabled": false, }) assert.NoError(t, err) @@ -107,12 +107,12 @@ func TestConfigNetInfoEnabled(t *testing.T) { assert.NotNil(t, v) v, err = newEvent.GetValue("host.ip") - assert.NoError(t, err) - assert.NotNil(t, v) + assert.Error(t, err) + assert.Nil(t, v) v, err = newEvent.GetValue("host.mac") - assert.NoError(t, err) - assert.NotNil(t, v) + assert.Error(t, err) + assert.Nil(t, v) } func TestConfigName(t *testing.T) { diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 6a3940019c9..76e3709ff2f 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -33,7 +33,7 @@ type Config struct { func defaultConfig() Config { return Config{ - NetInfoEnabled: false, + NetInfoEnabled: true, CacheTTL: 5 * time.Minute, } } diff --git a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc index bd3d47d8e86..76d5be26334 100644 --- a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc +++ b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc @@ -5,7 +5,6 @@ ------------------------------------------------------------------------------- processors: - add_host_metadata: - netinfo.enabled: false cache.ttl: 5m geo: name: nyc-dc1-rack1 @@ -19,7 +18,7 @@ processors: It has the following settings: -`netinfo.enabled`:: (Optional) Default false. Include IP addresses and MAC addresses as fields host.ip and host.mac +`netinfo.enabled`:: (Optional) Default true. Include IP addresses and MAC addresses as fields host.ip and host.mac `cache.ttl`:: (Optional) The processor uses an internal cache for the host metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether. diff --git a/libbeat/processors/add_observer_metadata/add_observer_metadata_test.go b/libbeat/processors/add_observer_metadata/add_observer_metadata_test.go index e686388ccc6..c9d55318fdd 100644 --- a/libbeat/processors/add_observer_metadata/add_observer_metadata_test.go +++ b/libbeat/processors/add_observer_metadata/add_observer_metadata_test.go @@ -42,12 +42,12 @@ func TestConfigDefault(t *testing.T) { assert.NoError(t, err) v, err := newEvent.GetValue("observer.ip") - assert.Error(t, err) - assert.Nil(t, v) + assert.NoError(t, err) + assert.NotNil(t, v) v, err = newEvent.GetValue("observer.mac") - assert.Error(t, err) - assert.Nil(t, v) + assert.NoError(t, err) + assert.NotNil(t, v) } func TestOverwriteFalse(t *testing.T) { @@ -86,13 +86,13 @@ func TestOverwriteTrue(t *testing.T) { assert.NotNil(t, v) } -func TestConfigNetInfoEnabled(t *testing.T) { +func TestConfigNetInfoDisabled(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{}, Timestamp: time.Now(), } testConfig, err := common.NewConfigFrom(map[string]interface{}{ - "netinfo.enabled": true, + "netinfo.enabled": false, }) assert.NoError(t, err) @@ -102,12 +102,12 @@ func TestConfigNetInfoEnabled(t *testing.T) { assert.NoError(t, err) v, err := newEvent.GetValue("observer.ip") - assert.NoError(t, err) - assert.NotNil(t, v) + assert.Error(t, err) + assert.Nil(t, v) v, err = newEvent.GetValue("observer.mac") - assert.NoError(t, err) - assert.NotNil(t, v) + assert.Error(t, err) + assert.Nil(t, v) } func TestConfigGeoEnabled(t *testing.T) { diff --git a/libbeat/processors/add_observer_metadata/config.go b/libbeat/processors/add_observer_metadata/config.go index 111ed9a3f1d..aea1fff2c1d 100644 --- a/libbeat/processors/add_observer_metadata/config.go +++ b/libbeat/processors/add_observer_metadata/config.go @@ -33,7 +33,7 @@ type Config struct { func defaultConfig() Config { return Config{ - NetInfoEnabled: false, + NetInfoEnabled: true, CacheTTL: 5 * time.Minute, } } diff --git a/libbeat/processors/add_observer_metadata/docs/add_observer_metadata.asciidoc b/libbeat/processors/add_observer_metadata/docs/add_observer_metadata.asciidoc index 1bf3e12eab7..5f9e7e87dbc 100644 --- a/libbeat/processors/add_observer_metadata/docs/add_observer_metadata.asciidoc +++ b/libbeat/processors/add_observer_metadata/docs/add_observer_metadata.asciidoc @@ -7,7 +7,6 @@ beta[] ------------------------------------------------------------------------------- processors: - add_observer_metadata: - netinfo.enabled: false cache.ttl: 5m geo: name: nyc-dc1-rack1 @@ -21,7 +20,7 @@ processors: It has the following settings: -`netinfo.enabled`:: (Optional) Default false. Include IP addresses and MAC addresses as fields observer.ip and observer.mac +`netinfo.enabled`:: (Optional) Default true. Include IP addresses and MAC addresses as fields observer.ip and observer.mac `cache.ttl`:: (Optional) The processor uses an internal cache for the observer metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 01f17b9445c..26e67530683 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1004,8 +1004,7 @@ metricbeat.modules: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index ad66e8fa18b..5537a683b3c 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -742,8 +742,7 @@ packetbeat.ignore_outgoing: false # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 51aaa89ec97..e22fe2cb0d4 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -238,8 +238,7 @@ winlogbeat.event_logs: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 8f70d82821e..3c78acd5449 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -371,8 +371,7 @@ auditbeat.modules: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 7b745fa0e09..36110d49d20 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1566,8 +1566,7 @@ filebeat.inputs: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index e6052c2822c..45d0e0bd349 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -581,8 +581,7 @@ functionbeat.provider.gcp.functions: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 28ce31cc400..987f0d8ad5e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1237,8 +1237,7 @@ metricbeat.modules: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index bcdbf155a04..7836a1e02ba 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -241,8 +241,7 @@ winlogbeat.event_logs: # The following example enriches each event with host metadata. # #processors: -#- add_host_metadata: -# netinfo.enabled: false +#- add_host_metadata: ~ # # The following example enriches each event with process metadata using # process IDs included in the event.