From 98bf114ff5a1b42d27dab7f9958e84f9804fc719 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 29 Apr 2021 16:45:34 +0800 Subject: [PATCH 1/6] libbeat: support explicit dynamic templates Add the field `DynamicTemplate bool` to `mapping.Field`, indicating that the field represents an explicitly named dynamic template. Update Elasticsearch template generation to create dynamic templates from these fields when the target Elasticsearch is 7.13.0+, when support for these dynamic templates was added and the requirement of match criteria was dropped. Such dynamic templates are suitable only for use in dynamic_templates bulk request parameters and in ingest pipelines; they will have no path or type matching criteria. --- libbeat/mapping/field.go | 8 ++++++ libbeat/template/processor.go | 45 ++++++++++++++++++++---------- libbeat/template/processor_test.go | 25 +++++++++++++++-- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go index eef0f6815796..7a78a0279108 100644 --- a/libbeat/mapping/field.go +++ b/libbeat/mapping/field.go @@ -56,6 +56,14 @@ type Field struct { MigrationAlias bool `config:"migration"` Dimension *bool `config:"dimension"` + // DynamicTemplate controls whether this field represents an explicitly + // named dynamic template. + // + // Such dynamic templates are only suitable for use in dynamic_template + // parameter in bulk requests or in ingest pipelines, as they will have + // no path or type match criteria. + DynamicTemplate bool `config:"dynamic_template"` + ObjectType string `config:"object_type"` ObjectTypeMappingType string `config:"object_type_mapping_type"` ScalingFactor int `config:"scaling_factor"` diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 6a6add512fa9..ace35ad807cb 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -120,7 +120,15 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com } if len(indexMapping) > 0 { - output.Put(mapping.GenerateKey(field.Name), indexMapping) + if field.DynamicTemplate { + // Explicit dynamic templates were introduced in + // Elasticsearch 7.13, ignore if unsupported + if !p.EsVersion.LessThan(common.MustNewVersion("7.13.0")) { + p.addDynamicTemplate(field.Name, "", "", indexMapping) + } + } else { + output.Put(mapping.GenerateKey(field.Name), indexMapping) + } } } return nil @@ -375,8 +383,11 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(f.ObjectTypeParams) != 0 { otParams = f.ObjectTypeParams } else { - otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{ - ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} + otParams = []mapping.ObjectTypeCfg{{ + ObjectType: f.ObjectType, + ObjectTypeMappingType: f.ObjectTypeMappingType, + ScalingFactor: f.ScalingFactor, + }} } for _, otp := range otParams { @@ -425,7 +436,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(otParams) > 1 { path = fmt.Sprintf("%s_%s", path, matchingType) } - p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType) + p.addDynamicTemplate(path, pathMatch, matchingType, dynProperties) } properties := getDefaultProperties(f) @@ -442,14 +453,14 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { } type dynamicTemplateKey struct { - path string + name string pathMatch string matchType string } -func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { +func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) bool { key := dynamicTemplateKey{ - path: path, + name: name, pathMatch: pathMatch, matchType: matchType, } @@ -458,21 +469,25 @@ func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties } else { if _, ok := p.dynamicTemplatesMap[key]; ok { // Dynamic template already added. - return + return false } } + dynamicTemplateProperties := common.MapStr{ + "mapping": properties, + } + if matchType != "" { + dynamicTemplateProperties["match_mapping_type"] = matchType + } + if pathMatch != "" { + dynamicTemplateProperties["path_match"] = pathMatch + } dynamicTemplate := common.MapStr{ - // Set the path of the field as name - path: common.MapStr{ - "mapping": properties, - "match_mapping_type": matchType, - "path_match": pathMatch, - }, + name: dynamicTemplateProperties, } p.dynamicTemplatesMap[key] = dynamicTemplate p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate) + return true } - func getDefaultProperties(f *mapping.Field) common.MapStr { // Currently no defaults exist properties := common.MapStr{} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 1fb3cfb08e07..85c87f9bbc0f 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/mapping" @@ -465,6 +466,20 @@ func TestDynamicTemplates(t *testing.T) { }, }, }, + { + field: mapping.Field{ + Name: "dynamic_histogram", + Type: "histogram", + DynamicTemplate: true, + }, + expected: []common.MapStr{ + { + "dynamic_histogram": common.MapStr{ + "mapping": common.MapStr{"type": "histogram"}, + }, + }, + }, + }, } for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { @@ -492,9 +507,13 @@ func TestDynamicTemplates(t *testing.T) { } for _, test := range tests { - p := &Processor{} - p.object(&test.field) - p.object(&test.field) // should not be added twice + output := make(common.MapStr) + p := &Processor{EsVersion: *common.MustNewVersion("8.0.0")} + err := p.Process(mapping.Fields{ + test.field, + test.field, // should not be added twice + }, &fieldState{Path: test.field.Path}, output) + require.NoError(t, err) assert.Equal(t, test.expected, p.dynamicTemplates) } } From 3476dbea296007520d4d22ad63760da628e7ed6d Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 29 Apr 2021 17:54:02 +0800 Subject: [PATCH 2/6] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 554e207e4156..0d3ca2ed3dcf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -607,6 +607,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115] +- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422] *Auditbeat* From 6ac7dab45b421600738e8ae4be501efb55d35469 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Fri, 30 Apr 2021 11:01:37 +0800 Subject: [PATCH 3/6] Revert signature change --- libbeat/template/processor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index ace35ad807cb..25b55a8ca854 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -458,7 +458,7 @@ type dynamicTemplateKey struct { matchType string } -func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) bool { +func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) { key := dynamicTemplateKey{ name: name, pathMatch: pathMatch, @@ -469,7 +469,7 @@ func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, proper } else { if _, ok := p.dynamicTemplatesMap[key]; ok { // Dynamic template already added. - return false + return } } dynamicTemplateProperties := common.MapStr{ @@ -486,8 +486,8 @@ func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, proper } p.dynamicTemplatesMap[key] = dynamicTemplate p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate) - return true } + func getDefaultProperties(f *mapping.Field) common.MapStr { // Currently no defaults exist properties := common.MapStr{} From 4efc92fe26d17f82d21433d267a223e70746dd72 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Fri, 30 Apr 2021 11:18:01 +0800 Subject: [PATCH 4/6] Disallow dynamic_template with object_type_params --- libbeat/mapping/field.go | 9 ++++++++- libbeat/mapping/field_test.go | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go index 7a78a0279108..09de581205d4 100644 --- a/libbeat/mapping/field.go +++ b/libbeat/mapping/field.go @@ -146,7 +146,14 @@ func (f *Field) validateType() error { if f.Format != "" { return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format) } - case "object", "group", "nested", "flattened": + case "object": + if f.DynamicTemplate && (len(f.ObjectTypeParams) > 0 || f.ObjectType != "") { + // When either ObjectTypeParams or ObjectType are set for an object-type field, + // libbeat/template will create dynamic templates. It does not make sense to + // use these with explicit dynamic templates. + return errors.New("dynamic_template not supported with object_type_params") + } + case "group", "nested", "flattened": // No check for them yet case "": // Module keys, not used as fields diff --git a/libbeat/mapping/field_test.go b/libbeat/mapping/field_test.go index 31a9b6c36844..c549c1b18e25 100644 --- a/libbeat/mapping/field_test.go +++ b/libbeat/mapping/field_test.go @@ -332,6 +332,20 @@ func TestFieldValidate(t *testing.T) { "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, + "invalid config mixing dynamic_template with object_type": { + cfg: common.MapStr{"dynamic_template": true, "type": "object", "object_type": "text"}, + err: true, + }, + "invalid config mixing dynamic_template with object_type_params": { + cfg: common.MapStr{ + "type": "object", + "object_type_params": []common.MapStr{{ + "object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100, + }}, + "dynamic_template": true, + }, + err: true, + }, } for name, test := range tests { From 4f0f5d9a42feda01b2ebd80d22252fe2f4f3facb Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 5 May 2021 12:49:12 +0800 Subject: [PATCH 5/6] Add a constant for min version --- libbeat/template/processor.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 47d389a9c535..2cd06f6d94e4 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -27,10 +27,11 @@ import ( ) var ( - minVersionAlias = common.MustNewVersion("6.4.0") - minVersionFieldMeta = common.MustNewVersion("7.6.0") - minVersionHistogram = common.MustNewVersion("7.6.0") - minVersionWildcard = common.MustNewVersion("7.9.0") + minVersionAlias = common.MustNewVersion("6.4.0") + minVersionFieldMeta = common.MustNewVersion("7.6.0") + minVersionHistogram = common.MustNewVersion("7.6.0") + minVersionWildcard = common.MustNewVersion("7.9.0") + minVersionExplicitDynamicTemplate = common.MustNewVersion("7.13.0") ) // Processor struct to process fields to template @@ -130,7 +131,7 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com if field.DynamicTemplate { // Explicit dynamic templates were introduced in // Elasticsearch 7.13, ignore if unsupported - if !p.EsVersion.LessThan(common.MustNewVersion("7.13.0")) { + if !p.EsVersion.LessThan(minVersionExplicitDynamicTemplate) { p.addDynamicTemplate(field.Name, "", "", indexMapping) } } else { From a2570c3630731ce97dee4c2f0929209e9aa3c161 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 5 May 2021 12:56:22 +0800 Subject: [PATCH 6/6] libbeat/mapping: fix field validation --- libbeat/mapping/field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go index f6b0d699ad5c..72ac861581d6 100644 --- a/libbeat/mapping/field.go +++ b/libbeat/mapping/field.go @@ -154,9 +154,7 @@ func (f *Field) validateType() error { case "date_range": allowedFormatters = []string{"date_range"} case "boolean", "binary", "ip", "alias", "array": - if f.Format != "" { - return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format) - } + // No formatters, metric types, or units allowed. case "object": if f.DynamicTemplate && (len(f.ObjectTypeParams) > 0 || f.ObjectType != "") { // When either ObjectTypeParams or ObjectType are set for an object-type field, @@ -164,6 +162,8 @@ func (f *Field) validateType() error { // use these with explicit dynamic templates. return errors.New("dynamic_template not supported with object_type_params") } + // No further checks for object yet. + return nil case "group", "nested", "flattened": // No check for them yet return nil