diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 971fa079ff0..faf993c5ad1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -189,6 +189,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add builder support for autodiscover and annotations builder {pull}6408[6408] - Add plugin support for autodiscover builders, providers {pull}6457[6457] - Preserve runtime from container statuses in Kubernetes autodiscover {pull}6456[6456] +- Experimental feature setup.template.append_fields added. {pull}6024[6024] *Auditbeat* diff --git a/libbeat/common/field.go b/libbeat/common/field.go index d8b4e2421b9..7a4b7e5a88b 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -99,6 +99,44 @@ func (f Fields) HasKey(key string) bool { return f.hasKey(keys) } +// HasNode checks if inside fields the given node exists +// In contrast to HasKey it not only compares the leaf nodes but +// every single key it traverses. +func (f Fields) HasNode(key string) bool { + keys := strings.Split(key, ".") + return f.hasNode(keys) +} + +func (f Fields) hasNode(keys []string) bool { + + // Nothing to compare, so does not contain it + if len(keys) == 0 { + return false + } + + key := keys[0] + keys = keys[1:] + + for _, field := range f { + + if field.Name == key { + + //// It's the last key to compare + if len(keys) == 0 { + return true + } + + // It's the last field to compare + if len(field.Fields) == 0 { + return true + } + + return field.Fields.hasNode(keys) + } + } + return false +} + // Recursively generates the correct key based on the dots // The mapping requires "properties" between each layer. This is added here. func GenerateKey(key string) string { @@ -134,3 +172,27 @@ func (f Fields) hasKey(keys []string) bool { } return false } + +// GetKeys returns a flat list of keys this Fields contains +func (f Fields) GetKeys() []string { + return f.getKeys("") +} + +func (f Fields) getKeys(namespace string) []string { + + var keys []string + + for _, field := range f { + fieldName := namespace + "." + field.Name + if namespace == "" { + fieldName = field.Name + } + if len(field.Fields) == 0 { + keys = append(keys, fieldName) + } else { + keys = append(keys, field.Fields.getKeys(fieldName)...) + } + } + + return keys +} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go index 2bab75a67d6..6c128b9bc9e 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/common/field_test.go @@ -68,7 +68,7 @@ func TestDynamicYaml(t *testing.T) { }{ { input: []byte(` -name: test +name: test dynamic: true`), output: Field{ Name: "test", @@ -115,3 +115,60 @@ dynamic: "strict"`), } } } + +func TestGetKeys(t *testing.T) { + tests := []struct { + fields Fields + keys []string + }{ + { + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + keys: []string{"test.find"}, + }, + { + fields: Fields{ + Field{ + Name: "a", Fields: Fields{ + Field{ + Name: "b", + }, + }, + }, + Field{ + Name: "a", Fields: Fields{ + Field{ + Name: "c", + }, + }, + }, + }, + keys: []string{"a.b", "a.c"}, + }, + { + fields: Fields{ + Field{ + Name: "a", + }, + Field{ + Name: "b", + }, + Field{ + Name: "c", + }, + }, + keys: []string{"a", "b", "c"}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.keys, test.fields.GetKeys()) + } +} diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 68e5f5612af..558f92c1b80 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -82,3 +82,7 @@ setup.template.overwrite: false setup.template.settings: _source.enabled: false ---------------------------------------------------------------------- + +*`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[] + +NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern. diff --git a/libbeat/template/config.go b/libbeat/template/config.go index eec7e6dab57..2db4fda34e7 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,12 +1,15 @@ package template +import "github.com/elastic/beats/libbeat/common" + type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Pattern string `config:"pattern"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Pattern string `config:"pattern"` + Fields string `config:"fields"` + AppendFields common.Fields `config:"append_fields"` + Overwrite bool `config:"overwrite"` + Settings TemplateSettings `config:"settings"` } type TemplateSettings struct { diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 31420d5f10c..d19e2849ae9 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -6,6 +6,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/common/fmtstr" ) @@ -24,7 +25,7 @@ type Template struct { pattern string beatVersion common.Version esVersion common.Version - settings TemplateSettings + config TemplateConfig } // New creates a new template instance @@ -87,17 +88,26 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC name: name, beatVersion: *bV, esVersion: *esV, - settings: config.Settings, + config: config, }, nil } // Load the given input and generates the input based on it func (t *Template) Load(file string) (common.MapStr, error) { + fields, err := common.LoadFieldsYaml(file) if err != nil { return nil, err } + if len(t.config.AppendFields) > 0 { + cfgwarn.Experimental("append_fields is used.") + fields, err = appendFields(fields, t.config.AppendFields) + if err != nil { + return nil, err + } + } + // Start processing at the root properties := common.MapStr{} processor := Processor{EsVersion: t.esVersion} @@ -155,7 +165,7 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards) } - indexSettings.DeepUpdate(t.settings.Index) + indexSettings.DeepUpdate(t.config.Settings.Index) var mappingName string if t.esVersion.Major >= 6 { @@ -182,9 +192,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. }, } - if len(t.settings.Source) > 0 { + if len(t.config.Settings.Source) > 0 { key := fmt.Sprintf("mappings.%s._source", mappingName) - basicStructure.Put(key, t.settings.Source) + basicStructure.Put(key, t.config.Settings.Source) } // ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009 @@ -200,3 +210,19 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. return basicStructure } + +func appendFields(fields, appendFields common.Fields) (common.Fields, error) { + if len(appendFields) > 0 { + appendFieldKeys := appendFields.GetKeys() + + // Append is only allowed to add fields, not overwrite + for _, key := range appendFieldKeys { + if fields.HasNode(key) { + return nil, fmt.Errorf("append_fields contains an already existing key: %s", key) + } + } + // Appends fields to existing fields + fields = append(fields, appendFields...) + } + return fields, nil +} diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 2e5200f73ad..191d4b07786 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" ) func TestNumberOfRoutingShards(t *testing.T) { @@ -54,3 +56,106 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) { assert.Equal(t, 5, shards.(int)) } + +func TestAppendFields(t *testing.T) { + tests := []struct { + fields common.Fields + appendFields common.Fields + error bool + }{ + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "b", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: false, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "b", + }, + common.Field{ + Name: "c", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: true, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: true, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + }, + }, + error: true, + }, + } + + for _, test := range tests { + _, err := appendFields(test.fields, test.appendFields) + if test.error { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + } +}