From b6408327490b87870c353b3376ac7865c8df4da0 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 13 Mar 2018 15:19:42 +0100 Subject: [PATCH] Add support for setting fields in the config file (#6024) * Add support for setting fields in the config file So far if the user wanted to modify the generated template, he either had to create his own template.json file or modify our `fields.yml` file. The problem with changing the `fields.yml` file is that with new versions it was hard to keep these up-to-date. This change allows to specify the few fields which should be set as part of the config. Setting fields is especially useful in the case of Metricbeat for modules like Http or Prometheus where the data is user specific and we don't know the structure in advance. This change also has affects on the generation of the index pattern in Kibana. The configuration looks as following: ``` setup.template.append_fields: - name: test.name type: keyword - name: test.hostname type: long ``` I would have preferred to use `setup.template.fields:` but that is already taken by the path to the file. Notes: * For this change to happen the template and index pattern must be overwritten * Overwriting existing fields is not allowed --- CHANGELOG.asciidoc | 1 + libbeat/common/field.go | 62 +++++++++++++++ libbeat/common/field_test.go | 59 ++++++++++++++- libbeat/docs/template-config.asciidoc | 4 + libbeat/template/config.go | 15 ++-- libbeat/template/template.go | 36 +++++++-- libbeat/template/template_test.go | 105 ++++++++++++++++++++++++++ 7 files changed, 270 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 971fa079ff0..faf993c5ad1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -189,6 +189,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add builder support for autodiscover and annotations builder {pull}6408[6408] - Add plugin support for autodiscover builders, providers {pull}6457[6457] - Preserve runtime from container statuses in Kubernetes autodiscover {pull}6456[6456] +- Experimental feature setup.template.append_fields added. {pull}6024[6024] *Auditbeat* diff --git a/libbeat/common/field.go b/libbeat/common/field.go index d8b4e2421b9..7a4b7e5a88b 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -99,6 +99,44 @@ func (f Fields) HasKey(key string) bool { return f.hasKey(keys) } +// HasNode checks if inside fields the given node exists +// In contrast to HasKey it not only compares the leaf nodes but +// every single key it traverses. +func (f Fields) HasNode(key string) bool { + keys := strings.Split(key, ".") + return f.hasNode(keys) +} + +func (f Fields) hasNode(keys []string) bool { + + // Nothing to compare, so does not contain it + if len(keys) == 0 { + return false + } + + key := keys[0] + keys = keys[1:] + + for _, field := range f { + + if field.Name == key { + + //// It's the last key to compare + if len(keys) == 0 { + return true + } + + // It's the last field to compare + if len(field.Fields) == 0 { + return true + } + + return field.Fields.hasNode(keys) + } + } + return false +} + // Recursively generates the correct key based on the dots // The mapping requires "properties" between each layer. This is added here. func GenerateKey(key string) string { @@ -134,3 +172,27 @@ func (f Fields) hasKey(keys []string) bool { } return false } + +// GetKeys returns a flat list of keys this Fields contains +func (f Fields) GetKeys() []string { + return f.getKeys("") +} + +func (f Fields) getKeys(namespace string) []string { + + var keys []string + + for _, field := range f { + fieldName := namespace + "." + field.Name + if namespace == "" { + fieldName = field.Name + } + if len(field.Fields) == 0 { + keys = append(keys, fieldName) + } else { + keys = append(keys, field.Fields.getKeys(fieldName)...) + } + } + + return keys +} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go index 2bab75a67d6..6c128b9bc9e 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/common/field_test.go @@ -68,7 +68,7 @@ func TestDynamicYaml(t *testing.T) { }{ { input: []byte(` -name: test +name: test dynamic: true`), output: Field{ Name: "test", @@ -115,3 +115,60 @@ dynamic: "strict"`), } } } + +func TestGetKeys(t *testing.T) { + tests := []struct { + fields Fields + keys []string + }{ + { + fields: Fields{ + Field{ + Name: "test", Fields: Fields{ + Field{ + Name: "find", + }, + }, + }, + }, + keys: []string{"test.find"}, + }, + { + fields: Fields{ + Field{ + Name: "a", Fields: Fields{ + Field{ + Name: "b", + }, + }, + }, + Field{ + Name: "a", Fields: Fields{ + Field{ + Name: "c", + }, + }, + }, + }, + keys: []string{"a.b", "a.c"}, + }, + { + fields: Fields{ + Field{ + Name: "a", + }, + Field{ + Name: "b", + }, + Field{ + Name: "c", + }, + }, + keys: []string{"a", "b", "c"}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.keys, test.fields.GetKeys()) + } +} diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 68e5f5612af..558f92c1b80 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -82,3 +82,7 @@ setup.template.overwrite: false setup.template.settings: _source.enabled: false ---------------------------------------------------------------------- + +*`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[] + +NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern. diff --git a/libbeat/template/config.go b/libbeat/template/config.go index eec7e6dab57..2db4fda34e7 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,12 +1,15 @@ package template +import "github.com/elastic/beats/libbeat/common" + type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Pattern string `config:"pattern"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Pattern string `config:"pattern"` + Fields string `config:"fields"` + AppendFields common.Fields `config:"append_fields"` + Overwrite bool `config:"overwrite"` + Settings TemplateSettings `config:"settings"` } type TemplateSettings struct { diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 31420d5f10c..d19e2849ae9 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -6,6 +6,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/common/fmtstr" ) @@ -24,7 +25,7 @@ type Template struct { pattern string beatVersion common.Version esVersion common.Version - settings TemplateSettings + config TemplateConfig } // New creates a new template instance @@ -87,17 +88,26 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC name: name, beatVersion: *bV, esVersion: *esV, - settings: config.Settings, + config: config, }, nil } // Load the given input and generates the input based on it func (t *Template) Load(file string) (common.MapStr, error) { + fields, err := common.LoadFieldsYaml(file) if err != nil { return nil, err } + if len(t.config.AppendFields) > 0 { + cfgwarn.Experimental("append_fields is used.") + fields, err = appendFields(fields, t.config.AppendFields) + if err != nil { + return nil, err + } + } + // Start processing at the root properties := common.MapStr{} processor := Processor{EsVersion: t.esVersion} @@ -155,7 +165,7 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards) } - indexSettings.DeepUpdate(t.settings.Index) + indexSettings.DeepUpdate(t.config.Settings.Index) var mappingName string if t.esVersion.Major >= 6 { @@ -182,9 +192,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. }, } - if len(t.settings.Source) > 0 { + if len(t.config.Settings.Source) > 0 { key := fmt.Sprintf("mappings.%s._source", mappingName) - basicStructure.Put(key, t.settings.Source) + basicStructure.Put(key, t.config.Settings.Source) } // ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009 @@ -200,3 +210,19 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. return basicStructure } + +func appendFields(fields, appendFields common.Fields) (common.Fields, error) { + if len(appendFields) > 0 { + appendFieldKeys := appendFields.GetKeys() + + // Append is only allowed to add fields, not overwrite + for _, key := range appendFieldKeys { + if fields.HasNode(key) { + return nil, fmt.Errorf("append_fields contains an already existing key: %s", key) + } + } + // Appends fields to existing fields + fields = append(fields, appendFields...) + } + return fields, nil +} diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 2e5200f73ad..191d4b07786 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" ) func TestNumberOfRoutingShards(t *testing.T) { @@ -54,3 +56,106 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) { assert.Equal(t, 5, shards.(int)) } + +func TestAppendFields(t *testing.T) { + tests := []struct { + fields common.Fields + appendFields common.Fields + error bool + }{ + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "b", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: false, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "b", + }, + common.Field{ + Name: "c", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: true, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + error: true, + }, + { + fields: common.Fields{ + common.Field{ + Name: "a", + Fields: common.Fields{ + common.Field{ + Name: "c", + }, + }, + }, + }, + appendFields: common.Fields{ + common.Field{ + Name: "a", + }, + }, + error: true, + }, + } + + for _, test := range tests { + _, err := appendFields(test.fields, test.appendFields) + if test.error { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + } +}