Merge remote-tracking branch 'upstream/master' into feature/module-at…

…-stage-level

* upstream/master:
  [CI] Fix permissions should not fail (elastic#18899)
  Revert "Allow the Docker image to be run with a random user id (elastic#12905)" (elastic#18872)
  Add new fields to HAProxy module of Metricbeat (elastic#18523)
  Avoid duplicate names in dynamic_templates (elastic#18849)

CHANGELOG.next.asciidoc

@@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938]
 - Remove the non-ECS `agent.hostname` field. Use the `agent.name` or `agent.id` fields for an identifier. {issue}16377[16377] {pull}18328[18328]
 - Make error message about locked data path actionable. {pull}18667[18667]
+- Ensure dynamic template names are unique for the same field. {pull}18849[18849]
 
 *Auditbeat*
 
@@ -265,7 +266,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 - Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808]
 - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
-- Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905]
 - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
 
 *Auditbeat*
@@ -453,6 +453,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add client address to events from http server module {pull}18336[18336]
 - Remove required for region/zone and make stackdriver a metricset in googlecloud. {issue}16785[16785] {pull}18398[18398]
 - Add memory metrics into compute googlecloud. {pull}18802[18802]
+- Add new fields to HAProxy module. {issue}18523[18523]
 
 *Packetbeat*
 

Jenkinsfile

@@ -800,7 +800,7 @@ def fixPermissions(location) {
   sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
     source ./dev-tools/common.bash
     docker_setup
-    script/fix_permissions.sh ${location}""")
+    script/fix_permissions.sh ${location}""", returnStatus: true)
 }
 
 def makeTarget(Map args = [:]) {

dev-tools/packaging/package_test.go

@@ -186,13 +186,8 @@ func checkDocker(t *testing.T, file string) {
 	checkDockerEntryPoint(t, p, info)
 	checkDockerLabels(t, p, info, file)
 	checkDockerUser(t, p, info, *rootUserContainer)
-
-	// The configuration file in the Docker image is expected to be readable and writable by any user who belongs to
-	// the root group. This is done in order to allow the docker image to run on secured Kubernetes environment where
-	// the user ID used to run a container can't be known in advance.
-	checkConfigPermissionsWithMode(t, p, os.FileMode(0660))
-	checkManifestPermissionsWithMode(t, p, os.FileMode(0660))
-
+	checkConfigPermissionsWithMode(t, p, os.FileMode(0640))
+	checkManifestPermissionsWithMode(t, p, os.FileMode(0640))
 	checkModulesPresent(t, "", p)
 	checkModulesDPresent(t, "", p)
 }

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -30,10 +30,10 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint
 RUN groupadd --gid 1000 {{ .BeatName }}
 
 RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
-    chown -R root:root {{ $beatHome }} && \
-    find {{ $beatHome }} -type d -exec chmod 0770 {} \; && \
-    find {{ $beatHome }} -type f -exec chmod 0660 {} \; && \
-    chmod 0770 {{ $beatBinary }} && \
+    chown -R root:{{ .BeatName }} {{ $beatHome }} && \
+    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 {{ $beatBinary }} && \
 {{- if .linux_capabilities }}
     setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
 {{- end }}
@@ -43,7 +43,7 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
     chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
 
 {{- if ne .user "root" }}
-RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
+RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}
 {{- end }}
 USER {{ .user }}
 

libbeat/template/processor.go

@@ -19,6 +19,7 @@ package template
 
 import (
 	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -319,29 +320,51 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 
 	for _, otp := range otParams {
 		dynProperties := getDefaultProperties(f)
+		var matchingType string
 
 		switch otp.ObjectType {
 		case "scaled_float":
 			dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor})
-			addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType))
+			matchingType = matchType("*", otp.ObjectTypeMappingType)
 		case "text":
 			dynProperties["type"] = "text"
 
 			if p.EsVersion.IsMajor(2) {
 				dynProperties["type"] = "string"
 				dynProperties["index"] = "analyzed"
 			}
-			addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
+			matchingType = matchType("string", otp.ObjectTypeMappingType)
 		case "keyword":
 			dynProperties["type"] = otp.ObjectType
-			addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
+			matchingType = matchType("string", otp.ObjectTypeMappingType)
 		case "byte", "double", "float", "long", "short", "boolean":
 			dynProperties["type"] = otp.ObjectType
-			addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType))
+			matchingType = matchType(otp.ObjectType, otp.ObjectTypeMappingType)
 		case "histogram":
 			dynProperties["type"] = otp.ObjectType
-			addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType))
+			matchingType = matchType("*", otp.ObjectTypeMappingType)
+		default:
+			continue
+		}
+
+		path := f.Path
+		if len(path) > 0 {
+			path += "."
 		}
+		path += f.Name
+		pathMatch := path
+		// ensure the `path_match` string ends with a `*`
+		if !strings.ContainsRune(path, '*') {
+			pathMatch += ".*"
+		}
+		// When multiple object type parameters are detected for a field,
+		// add a unique part to the name of the dynamic template.
+		// Duplicated dynamic template names can lead to errors when template
+		// inheritance is applied, and will not be supported in future versions
+		if len(otParams) > 1 {
+			path = fmt.Sprintf("%s_%s", path, matchingType)
+		}
+		addDynamicTemplate(path, pathMatch, dynProperties, matchingType)
 	}
 
 	properties := getDefaultProperties(f)
@@ -357,18 +380,10 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 	return properties
 }
 
-func addDynamicTemplate(f *mapping.Field, properties common.MapStr, matchType string) {
-	path := ""
-	if len(f.Path) > 0 {
-		path = f.Path + "."
-	}
-	pathMatch := path + f.Name
-	if !strings.ContainsRune(pathMatch, '*') {
-		pathMatch += ".*"
-	}
+func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) {
 	template := common.MapStr{
 		// Set the path of the field as name
-		path + f.Name: common.MapStr{
+		path: common.MapStr{
 			"mapping":            properties,
 			"match_mapping_type": matchType,
 			"path_match":         pathMatch,

libbeat/template/processor_test.go

@@ -443,22 +443,22 @@ func TestDynamicTemplates(t *testing.T) {
 				Name: "context",
 			},
 			expected: []common.MapStr{
-				common.MapStr{
-					"context": common.MapStr{
+				{
+					"context_float": common.MapStr{
 						"mapping":            common.MapStr{"type": "float"},
 						"match_mapping_type": "float",
 						"path_match":         "context.*",
 					},
 				},
-				common.MapStr{
-					"context": common.MapStr{
+				{
+					"context_boolean": common.MapStr{
 						"mapping":            common.MapStr{"type": "boolean"},
 						"match_mapping_type": "boolean",
 						"path_match":         "context.*",
 					},
 				},
-				common.MapStr{
-					"context": common.MapStr{
+				{
+					"context_*": common.MapStr{
 						"mapping":            common.MapStr{"type": "scaled_float", "scaling_factor": 10000},
 						"match_mapping_type": "*",
 						"path_match":         "context.*",