Add IP-addresses and MAC-addresses to event (#6878)

This is a pull request for issue #5396

CHANGELOG.asciidoc

@@ -118,6 +118,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Added logging of system info at Beat startup. {issue}5946[5946]
 - Do not log errors if X-Pack Monitoring is enabled but Elastisearch X-Pack is not. {pull}6627[6627]
 - Add rename processor. {pull}6292[6292]
+- Add IP-addresses and MAC-addresses to add_host_metadata. {pull}6878[6878]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -288,7 +288,13 @@ auditbeat.modules:
 #
 #processors:
 #- add_docker_metadata: ~
-#- add_host_metadata: ~
+#
+# The following example enriches each event with host metadata.
+#
+#processors:
+#- add_host_metadata:
+#   netinfo.enabled: false
+#
 
 #============================= Elastic Cloud ==================================
 

auditbeat/docs/fields.asciidoc

@@ -3258,6 +3258,26 @@ type: keyword
 OS family (e.g. redhat, debian, freebsd, windows).
 
 
+--
+
+*`host.ip`*::
++
+--
+type: ip
+
+List of IP-addresses.
+
+
+--
+
+*`host.mac`*::
++
+--
+type: keyword
+
+List of hardware-addresses, usually MAC-addresses.
+
+
 --
 
 [[exported-fields-kubernetes-processor]]

filebeat/docs/fields.asciidoc

@@ -851,6 +851,26 @@ type: keyword
 OS family (e.g. redhat, debian, freebsd, windows).
 
 
+--
+
+*`host.ip`*::
++
+--
+type: ip
+
+List of IP-addresses.
+
+
+--
+
+*`host.mac`*::
++
+--
+type: keyword
+
+List of hardware-addresses, usually MAC-addresses.
+
+
 --
 
 [[exported-fields-icinga]]

filebeat/filebeat.reference.yml

@@ -830,7 +830,13 @@ filebeat.inputs:
 #
 #processors:
 #- add_docker_metadata: ~
-#- add_host_metadata: ~
+#
+# The following example enriches each event with host metadata.
+#
+#processors:
+#- add_host_metadata:
+#   netinfo.enabled: false
+#
 
 #============================= Elastic Cloud ==================================
 

heartbeat/docs/fields.asciidoc

@@ -421,6 +421,26 @@ type: keyword
 OS family (e.g. redhat, debian, freebsd, windows).
 
 
+--
+
+*`host.ip`*::
++
+--
+type: ip
+
+List of IP-addresses.
+
+
+--
+
+*`host.mac`*::
++
+--
+type: keyword
+
+List of hardware-addresses, usually MAC-addresses.
+
+
 --
 
 [[exported-fields-http]]

heartbeat/heartbeat.reference.yml

@@ -397,7 +397,13 @@ heartbeat.scheduler:
 #
 #processors:
 #- add_docker_metadata: ~
-#- add_host_metadata: ~
+#
+# The following example enriches each event with host metadata.
+#
+#processors:
+#- add_host_metadata:
+#   netinfo.enabled: false
+#
 
 #============================= Elastic Cloud ==================================
 

libbeat/_meta/config.reference.yml

@@ -183,7 +183,13 @@
 #
 #processors:
 #- add_docker_metadata: ~
-#- add_host_metadata: ~
+#
+# The following example enriches each event with host metadata.
+#
+#processors:
+#- add_host_metadata:
+#   netinfo.enabled: false
+#
 
 #============================= Elastic Cloud ==================================
 

libbeat/docs/processors-using.asciidoc

@@ -727,6 +727,17 @@ forget metadata for a container, 60s by default.
 
 beta[]
 
+[source,yaml]
+-------------------------------------------------------------------------------
+processors:
+- add_host_metadata:
+    netinfo.enabled: false
+-------------------------------------------------------------------------------
+
+It has the following settings:
+
+`netinfo.enabled`:: (Optional) Default false. Include IP adresses and MAC addresses as fields host.ip and host.mac
+
 The `add_host_metadata` processor annotates each event with relevant metadata from the host machine.
 The fields added to the event are looking as following:
 
@@ -742,7 +753,9 @@ The fields added to the event are looking as following:
          "build":"16G1212",
          "platform":"darwin",
          "version":"10.12.6"
-      }
+      },
+      ip: ["192.168.0.1", "10.0.0.1"],
+      mac: ["00:25:96:12:34:56", "72:00:06:ff:79:f1"]
    }
 }
 -------------------------------------------------------------------------------

libbeat/processors/add_host_metadata/_meta/fields.yml

@@ -31,3 +31,12 @@
           type: keyword
           description: >
             OS family (e.g. redhat, debian, freebsd, windows).
+        - name: ip
+          type: ip
+          description: >
+            List of IP-addresses.
+        - name: mac
+          type: keyword
+          description: >
+            List of hardware-addresses, usually MAC-addresses.
+

libbeat/processors/add_host_metadata/add_host_metadata.go

@@ -1,10 +1,16 @@
 package add_host_metadata
 
 import (
+	"fmt"
+	"net"
 	"time"
 
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/processors"
 	"github.com/elastic/go-sysinfo"
 	"github.com/elastic/go-sysinfo/types"
@@ -18,19 +24,27 @@ type addHostMetadata struct {
 	info       types.HostInfo
 	lastUpdate time.Time
 	data       common.MapStr
+	config     Config
 }
 
 const (
+	processorName   = "add_host_metadata"
 	cacheExpiration = time.Minute * 5
 )
 
-func newHostMetadataProcessor(_ *common.Config) (processors.Processor, error) {
+func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) {
+	config := defaultConfig()
+	if err := cfg.Unpack(&config); err != nil {
+		return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName)
+	}
+
 	h, err := sysinfo.Host()
 	if err != nil {
 		return nil, err
 	}
 	p := &addHostMetadata{
-		info: h.Info(),
+		info:   h.Info(),
+		config: config,
 	}
 	return p, nil
 }
@@ -71,10 +85,72 @@ func (p *addHostMetadata) loadData() {
 		if p.info.OS.Build != "" {
 			p.data.Put("host.os.build", p.info.OS.Build)
 		}
+
+		if p.config.NetInfoEnabled {
+			// IP-address and MAC-address
+			var ipList, hwList, err = p.getNetInfo()
+			if err != nil {
+				logp.Info("Error when getting network information %v", err)
+			}
+
+			if len(ipList) > 0 {
+				p.data.Put("host.ip", ipList)
+			}
+			if len(hwList) > 0 {
+				p.data.Put("host.mac", hwList)
+			}
+		}
+
 		p.lastUpdate = time.Now()
 	}
 }
 
+func (p addHostMetadata) getNetInfo() ([]string, []string, error) {
+	var ipList []string
+	var hwList []string
+
+	// Get all interfaces and loop through them
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Keep track of all errors
+	var errs multierror.Errors
+
+	for _, i := range ifaces {
+		// Skip loopback interfaces
+		if i.Flags&net.FlagLoopback == net.FlagLoopback {
+			continue
+		}
+
+		hw := i.HardwareAddr.String()
+		// Skip empty hardware addresses
+		if hw != "" {
+			hwList = append(hwList, hw)
+		}
+
+		addrs, err := i.Addrs()
+		if err != nil {
+			// If we get an error, keep track of it and continue with the next interface
+			errs = append(errs, err)
+			continue
+		}
+
+		for _, addr := range addrs {
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ipList = append(ipList, v.IP.String())
+			case *net.IPAddr:
+				ipList = append(ipList, v.IP.String())
+			}
+		}
+	}
+
+	return ipList, hwList, errs.Err()
+}
+
 func (p addHostMetadata) String() string {
-	return "add_host_metadata=[]"
+	return fmt.Sprintf("%v=[netinfo.enabled=[%v]]",
+		processorName, p.config.NetInfoEnabled)
 }

libbeat/processors/add_host_metadata/add_host_metadata_test.go

@@ -13,12 +13,48 @@ import (
 	"github.com/elastic/go-sysinfo/types"
 )
 
-func TestRun(t *testing.T) {
+func TestConfigDefault(t *testing.T) {
 	event := &beat.Event{
 		Fields:    common.MapStr{},
 		Timestamp: time.Now(),
 	}
-	p, err := newHostMetadataProcessor(nil)
+	testConfig, err := common.NewConfigFrom(map[string]interface{}{})
+	assert.NoError(t, err)
+
+	p, err := newHostMetadataProcessor(testConfig)
+	if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
+		assert.IsType(t, types.ErrNotImplemented, err)
+		return
+	}
+	assert.NoError(t, err)
+
+	newEvent, err := p.Run(event)
+	assert.NoError(t, err)
+
+	v, err := newEvent.GetValue("host.os.family")
+	assert.NoError(t, err)
+	assert.NotNil(t, v)
+
+	v, err = newEvent.GetValue("host.ip")
+	assert.Error(t, err)
+	assert.Nil(t, v)
+
+	v, err = newEvent.GetValue("host.mac")
+	assert.Error(t, err)
+	assert.Nil(t, v)
+}
+
+func TestConfigNetInfoEnabled(t *testing.T) {
+	event := &beat.Event{
+		Fields:    common.MapStr{},
+		Timestamp: time.Now(),
+	}
+	testConfig, err := common.NewConfigFrom(map[string]interface{}{
+		"netinfo.enabled": true,
+	})
+	assert.NoError(t, err)
+
+	p, err := newHostMetadataProcessor(testConfig)
 	if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
 		assert.IsType(t, types.ErrNotImplemented, err)
 		return
@@ -31,4 +67,12 @@ func TestRun(t *testing.T) {
 	v, err := newEvent.GetValue("host.os.family")
 	assert.NoError(t, err)
 	assert.NotNil(t, v)
+
+	v, err = newEvent.GetValue("host.ip")
+	assert.NoError(t, err)
+	assert.NotNil(t, v)
+
+	v, err = newEvent.GetValue("host.mac")
+	assert.NoError(t, err)
+	assert.NotNil(t, v)
 }

libbeat/processors/add_host_metadata/config.go

@@ -0,0 +1,12 @@
+package add_host_metadata
+
+// Config for add_host_metadata processor.
+type Config struct {
+	NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event
+}
+
+func defaultConfig() Config {
+	return Config{
+		NetInfoEnabled: false,
+	}
+}