diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 3a3d73c7a13..8ffb7f62d11 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -256,6 +256,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Report configured queue type. {pull}8091[8091] - Enable `host` and `cloud` metadata processors by default. {pull}8596[8596] - Autodiscovery no longer requires that the `condition` field be set. If left unset all configs will be matched. {pull}9029[9029] +- Add DeDot method in add_docker_metadata processor in libbeat. {issue}9350[9350] {pull}9505[9505] *Filebeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index c28191f7f1e..2f67e3c7b10 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -286,6 +286,7 @@ auditbeat.modules: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/auditbeat/docs/getting-started.asciidoc b/auditbeat/docs/getting-started.asciidoc index a1b5f49f48e..6ec1dd9c729 100644 --- a/auditbeat/docs/getting-started.asciidoc +++ b/auditbeat/docs/getting-started.asciidoc @@ -97,6 +97,22 @@ endif::[] [[docker]] *docker:* +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + See <> for deploying Docker containers. [[win]] diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 5e750bb54a1..d4a35819e0e 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -965,6 +965,7 @@ filebeat.inputs: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 5235cea3309..a811525fbac 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -424,6 +424,7 @@ heartbeat.scheduler: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 6dc6ae01aae..408df907537 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -237,6 +237,7 @@ journalbeat.inputs: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index e49f917cb52..e3eb98b63c3 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -179,6 +179,7 @@ # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 5fcc7497d57..a9500e9498c 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -848,6 +848,7 @@ processors: #match_source_index: 4 #match_short_id: true #cleanup_timeout: 60 + #labels.dedot: false # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: # certificate_authority: "/etc/pki/root/ca.pem" @@ -885,6 +886,9 @@ for container ID. It defaults to 4 to match `cleanup_timeout`:: (Optional) Time of inactivity to consider we can clean and forget metadata for a container, 60s by default. +`labels.dedot`:: (Optional) Default to be false. If set to true, replace dots in + labels with `_`. + [[add-host-metadata]] === Add Host metadata diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 30d2ca064c1..1437df22095 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -60,6 +60,7 @@ type addDockerMetadata struct { pidFields []string // Field names that contain PIDs. cgroups *common.Cache // Cache of PID (int) to cgropus (map[string]string). hostFS string // Directory where /proc is found + dedot bool // If set to true, replace dots in labels with `_`. } func newDockerMetadataProcessor(cfg *common.Config) (processors.Processor, error) { @@ -103,6 +104,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker. sourceProcessor: sourceProcessor, pidFields: config.MatchPIDs, hostFS: config.HostFS, + dedot: config.DeDot, }, nil } @@ -174,7 +176,12 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { if len(container.Labels) > 0 { labels := common.MapStr{} for k, v := range container.Labels { - safemapstr.Put(labels, k, v) + if d.dedot { + label := common.DeDot(k) + labels.Put(label, v) + } else { + safemapstr.Put(labels, k, v) + } } meta.Put("container.labels", labels) } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 96c7de1ebb4..ba31b4919d4 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -148,6 +148,51 @@ func TestMatchContainer(t *testing.T) { }, result.Fields) } +func TestMatchContainerWithDedot(t *testing.T) { + testConfig, err := common.NewConfigFrom(map[string]interface{}{ + "match_fields": []string{"foo"}, + "labels.dedot": true, + }) + assert.NoError(t, err) + + p, err := buildDockerMetadataProcessor(testConfig, MockWatcherFactory( + map[string]*docker.Container{ + "container_id": &docker.Container{ + ID: "container_id", + Image: "image", + Name: "name", + Labels: map[string]string{ + "a.x": "1", + "b": "2", + "b.foo": "3", + }, + }, + })) + assert.NoError(t, err, "initializing add_docker_metadata processor") + + input := common.MapStr{ + "foo": "container_id", + } + result, err := p.Run(&beat.Event{Fields: input}) + assert.NoError(t, err, "processing an event") + + assert.EqualValues(t, common.MapStr{ + "docker": common.MapStr{ + "container": common.MapStr{ + "id": "container_id", + "image": "image", + "labels": common.MapStr{ + "a_x": "1", + "b": "2", + "b_foo": "3", + }, + "name": "name", + }, + }, + "foo": "container_id", + }, result.Fields) +} + func TestMatchSource(t *testing.T) { // Use defaults testConfig, err := common.NewConfigFrom(map[string]interface{}{}) diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index a88caa97d0c..dc1a3d4fc89 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -33,6 +33,7 @@ type Config struct { SourceIndex int `config:"match_source_index"` // Index in the source path split by / to look for container ID. MatchPIDs []string `config:"match_pids"` // A list of fields containing process IDs (PIDs). HostFS string `config:"system.hostfs"` // Specifies the mount point of the host’s filesystem for use in monitoring a host from within a container. + DeDot bool `config:"labels.dedot"` // If set to true, replace dots in labels with `_`. // Annotations are kept after container is killed, until they haven't been // accessed for a full `cleanup_timeout`: @@ -45,5 +46,6 @@ func defaultConfig() Config { MatchSource: true, SourceIndex: 4, // Use 4 to match the CID in /var/lib/docker/containers//*.log. MatchPIDs: []string{"process.pid", "process.ppid"}, + DeDot: false, } } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 96ffbeaef86..db6bfa13741 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -864,6 +864,7 @@ metricbeat.modules: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json index f878e335195..18f1062ebc4 100644 --- a/metricbeat/module/elasticsearch/node/_meta/data.json +++ b/metricbeat/module/elasticsearch/node/_meta/data.json @@ -47,4 +47,4 @@ "service": { "name": "elasticsearch" } -} \ No newline at end of file +} diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 93f144d76a3..63b062a1bb2 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -660,6 +660,7 @@ packetbeat.protocols: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 9e9a8a9657f..6597dc8687e 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -208,6 +208,7 @@ winlogbeat.event_logs: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index c48e48364d4..bb4d3c9a357 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -993,6 +993,7 @@ filebeat.inputs: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 88c5bac11ce..d6e9430ba8c 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -273,6 +273,7 @@ functionbeat.provider.aws.functions: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index f6b7ce9cf92..1f8f1a05ba9 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -872,6 +872,7 @@ metricbeat.modules: # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem"