Cherry-pick to 6.x: Add DeDot in add_docker_metadata processor (#9505)

CHANGELOG.asciidoc

@@ -256,6 +256,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits]
 - Report configured queue type. {pull}8091[8091]
 - Enable `host` and `cloud` metadata processors by default. {pull}8596[8596]
 - Autodiscovery no longer requires that the `condition` field be set. If left unset all configs will be matched. {pull}9029[9029]
+- Add DeDot method in add_docker_metadata processor in libbeat. {issue}9350[9350] {pull}9505[9505]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -286,6 +286,7 @@ auditbeat.modules:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

auditbeat/docs/getting-started.asciidoc

@@ -97,6 +97,22 @@ endif::[]
 [[docker]]
 *docker:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {stack-version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+["source","sh",subs="attributes"]
+------------------------------------------------
+curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz
+tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz
+------------------------------------------------
+
+endif::[]
+
 See <<running-on-docker, Running on Docker>> for deploying Docker containers.
 
 [[win]]

filebeat/filebeat.reference.yml

@@ -965,6 +965,7 @@ filebeat.inputs:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

heartbeat/heartbeat.reference.yml

@@ -424,6 +424,7 @@ heartbeat.scheduler:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

journalbeat/journalbeat.reference.yml

@@ -237,6 +237,7 @@ journalbeat.inputs:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

libbeat/_meta/config.reference.yml

@@ -179,6 +179,7 @@
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

libbeat/docs/processors-using.asciidoc

@@ -848,6 +848,7 @@ processors:
  #match_source_index: 4
  #match_short_id: true
  #cleanup_timeout: 60
+ #labels.dedot: false
  # To connect to Docker over TLS you must specify a client and CA certificate.
  #ssl:
  # certificate_authority: "/etc/pki/root/ca.pem"
@@ -885,6 +886,9 @@ for container ID. It defaults to 4 to match
 `cleanup_timeout`:: (Optional) Time of inactivity to consider we can clean and
 forget metadata for a container, 60s by default.
 
+`labels.dedot`:: (Optional) Default to be false. If set to true, replace dots in
+ labels with `_`.
+
 [[add-host-metadata]]
 === Add Host metadata
 

libbeat/processors/add_docker_metadata/add_docker_metadata.go

@@ -60,6 +60,7 @@ type addDockerMetadata struct {
  pidFields []string // Field names that contain PIDs.
  cgroups *common.Cache // Cache of PID (int) to cgropus (map[string]string).
  hostFS string // Directory where /proc is found
+ dedot bool // If set to true, replace dots in labels with `_`.
 }
 
 func newDockerMetadataProcessor(cfg *common.Config) (processors.Processor, error) {
@@ -103,6 +104,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
  sourceProcessor: sourceProcessor,
  pidFields: config.MatchPIDs,
  hostFS: config.HostFS,
+ dedot: config.DeDot,
  }, nil
 }
 
@@ -174,7 +176,12 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
  if len(container.Labels) > 0 {
  labels := common.MapStr{}
  for k, v := range container.Labels {
- safemapstr.Put(labels, k, v)
+ if d.dedot {
+ label := common.DeDot(k)
+ labels.Put(label, v)
+ } else {
+ safemapstr.Put(labels, k, v)
+ }
  }
  meta.Put("container.labels", labels)
  }

libbeat/processors/add_docker_metadata/add_docker_metadata_test.go

@@ -148,6 +148,51 @@ func TestMatchContainer(t *testing.T) {
  }, result.Fields)
 }
 
+func TestMatchContainerWithDedot(t *testing.T) {
+ testConfig, err := common.NewConfigFrom(map[string]interface{}{
+ "match_fields": []string{"foo"},
+ "labels.dedot": true,
+ })
+ assert.NoError(t, err)
+
+ p, err := buildDockerMetadataProcessor(testConfig, MockWatcherFactory(
+ map[string]*docker.Container{
+ "container_id": &docker.Container{
+ ID: "container_id",
+ Image: "image",
+ Name: "name",
+ Labels: map[string]string{
+ "a.x": "1",
+ "b": "2",
+ "b.foo": "3",
+ },
+ },
+ }))
+ assert.NoError(t, err, "initializing add_docker_metadata processor")
+
+ input := common.MapStr{
+ "foo": "container_id",
+ }
+ result, err := p.Run(&beat.Event{Fields: input})
+ assert.NoError(t, err, "processing an event")
+
+ assert.EqualValues(t, common.MapStr{
+ "docker": common.MapStr{
+ "container": common.MapStr{
+ "id": "container_id",
+ "image": "image",
+ "labels": common.MapStr{
+ "a_x": "1",
+ "b": "2",
+ "b_foo": "3",
+ },
+ "name": "name",
+ },
+ },
+ "foo": "container_id",
+ }, result.Fields)
+}
+
 func TestMatchSource(t *testing.T) {
  // Use defaults
  testConfig, err := common.NewConfigFrom(map[string]interface{}{})

libbeat/processors/add_docker_metadata/config.go

@@ -33,6 +33,7 @@ type Config struct {
  SourceIndex int `config:"match_source_index"` // Index in the source path split by / to look for container ID.
  MatchPIDs []string `config:"match_pids"` // A list of fields containing process IDs (PIDs).
  HostFS string `config:"system.hostfs"` // Specifies the mount point of the host’s filesystem for use in monitoring a host from within a container.
+ DeDot bool `config:"labels.dedot"` // If set to true, replace dots in labels with `_`.
 
  // Annotations are kept after container is killed, until they haven't been
  // accessed for a full `cleanup_timeout`:
@@ -45,5 +46,6 @@ func defaultConfig() Config {
  MatchSource: true,
  SourceIndex: 4, // Use 4 to match the CID in /var/lib/docker/containers/<container_id>/*.log.
  MatchPIDs: []string{"process.pid", "process.ppid"},
+ DeDot: false,
  }
 }

metricbeat/metricbeat.reference.yml

@@ -864,6 +864,7 @@ metricbeat.modules:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

metricbeat/module/elasticsearch/node/_meta/data.json

@@ -47,4 +47,4 @@
  "service": {
  "name": "elasticsearch"
  }
-}
+}

packetbeat/packetbeat.reference.yml

@@ -660,6 +660,7 @@ packetbeat.protocols:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

winlogbeat/winlogbeat.reference.yml

@@ -208,6 +208,7 @@ winlogbeat.event_logs:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

x-pack/filebeat/filebeat.reference.yml

@@ -993,6 +993,7 @@ filebeat.inputs:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

x-pack/functionbeat/functionbeat.reference.yml

@@ -273,6 +273,7 @@ functionbeat.provider.aws.functions:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"

x-pack/metricbeat/metricbeat.reference.yml

@@ -872,6 +872,7 @@ metricbeat.modules:
 # match_source_index: 4
 # match_short_id: false
 # cleanup_timeout: 60
+# labels.dedot: false
 # # To connect to Docker over TLS you must specify a client and CA certificate.
 # #ssl:
 # # certificate_authority: "/etc/pki/root/ca.pem"