Add support for setting fields in the config file (#6024)

* Add support for setting fields in the config file

So far if the user wanted to modify the generated template, he either had to create his own template.json file or modify our `fields.yml` file. The problem with changing the `fields.yml` file is that with new versions it was hard to keep these up-to-date. This change allows to specify the few fields which should be set as part of the config.

Setting fields is especially useful in the case of Metricbeat for modules like Http or Prometheus where the data is user specific and we don't know the structure in advance.

This change also has affects on the generation of the index pattern in Kibana.

The configuration looks as following:

```
setup.template.append_fields:
  - name: test.name
    type: keyword
  - name: test.hostname
    type: long
```

I would have preferred to use `setup.template.fields:` but that is already taken by the path to the file.

Notes:
* For this change to happen the template and index pattern must be overwritten
* Overwriting existing fields is not allowed

CHANGELOG.asciidoc

@@ -189,6 +189,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add builder support for autodiscover and annotations builder {pull}6408[6408]
 - Add plugin support for autodiscover builders, providers {pull}6457[6457]
 - Preserve runtime from container statuses in Kubernetes autodiscover {pull}6456[6456]
+- Experimental feature setup.template.append_fields added. {pull}6024[6024]
 
 *Auditbeat*
 

libbeat/common/field.go

@@ -99,6 +99,44 @@ func (f Fields) HasKey(key string) bool {
 	return f.hasKey(keys)
 }
 
+// HasNode checks if inside fields the given node exists
+// In contrast to HasKey it not only compares the leaf nodes but
+// every single key it traverses.
+func (f Fields) HasNode(key string) bool {
+	keys := strings.Split(key, ".")
+	return f.hasNode(keys)
+}
+
+func (f Fields) hasNode(keys []string) bool {
+
+	// Nothing to compare, so does not contain it
+	if len(keys) == 0 {
+		return false
+	}
+
+	key := keys[0]
+	keys = keys[1:]
+
+	for _, field := range f {
+
+		if field.Name == key {
+
+			//// It's the last key to compare
+			if len(keys) == 0 {
+				return true
+			}
+
+			// It's the last field to compare
+			if len(field.Fields) == 0 {
+				return true
+			}
+
+			return field.Fields.hasNode(keys)
+		}
+	}
+	return false
+}
+
 // Recursively generates the correct key based on the dots
 // The mapping requires "properties" between each layer. This is added here.
 func GenerateKey(key string) string {
@@ -134,3 +172,27 @@ func (f Fields) hasKey(keys []string) bool {
 	}
 	return false
 }
+
+// GetKeys returns a flat list of keys this Fields contains
+func (f Fields) GetKeys() []string {
+	return f.getKeys("")
+}
+
+func (f Fields) getKeys(namespace string) []string {
+
+	var keys []string
+
+	for _, field := range f {
+		fieldName := namespace + "." + field.Name
+		if namespace == "" {
+			fieldName = field.Name
+		}
+		if len(field.Fields) == 0 {
+			keys = append(keys, fieldName)
+		} else {
+			keys = append(keys, field.Fields.getKeys(fieldName)...)
+		}
+	}
+
+	return keys
+}

libbeat/common/field_test.go

@@ -68,7 +68,7 @@ func TestDynamicYaml(t *testing.T) {
 	}{
 		{
 			input: []byte(`
-name: test 
+name: test
 dynamic: true`),
 			output: Field{
 				Name:    "test",
@@ -115,3 +115,60 @@ dynamic: "strict"`),
 		}
 	}
 }
+
+func TestGetKeys(t *testing.T) {
+	tests := []struct {
+		fields Fields
+		keys   []string
+	}{
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			keys: []string{"test.find"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "a", Fields: Fields{
+						Field{
+							Name: "b",
+						},
+					},
+				},
+				Field{
+					Name: "a", Fields: Fields{
+						Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			keys: []string{"a.b", "a.c"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "a",
+				},
+				Field{
+					Name: "b",
+				},
+				Field{
+					Name: "c",
+				},
+			},
+			keys: []string{"a", "b", "c"},
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.keys, test.fields.GetKeys())
+	}
+}

libbeat/docs/template-config.asciidoc

@@ -82,3 +82,7 @@ setup.template.overwrite: false
 setup.template.settings:
   _source.enabled: false
 ----------------------------------------------------------------------
+
+*`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[]
+
+NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern.

libbeat/template/config.go

@@ -1,12 +1,15 @@
 package template
 
+import "github.com/elastic/beats/libbeat/common"
+
 type TemplateConfig struct {
-	Enabled   bool             `config:"enabled"`
-	Name      string           `config:"name"`
-	Pattern   string           `config:"pattern"`
-	Fields    string           `config:"fields"`
-	Overwrite bool             `config:"overwrite"`
-	Settings  TemplateSettings `config:"settings"`
+	Enabled      bool             `config:"enabled"`
+	Name         string           `config:"name"`
+	Pattern      string           `config:"pattern"`
+	Fields       string           `config:"fields"`
+	AppendFields common.Fields    `config:"append_fields"`
+	Overwrite    bool             `config:"overwrite"`
+	Settings     TemplateSettings `config:"settings"`
 }
 
 type TemplateSettings struct {

libbeat/template/template.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/common/fmtstr"
 )
 
@@ -24,7 +25,7 @@ type Template struct {
 	pattern     string
 	beatVersion common.Version
 	esVersion   common.Version
-	settings    TemplateSettings
+	config      TemplateConfig
 }
 
 // New creates a new template instance
@@ -87,17 +88,26 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC
 		name:        name,
 		beatVersion: *bV,
 		esVersion:   *esV,
-		settings:    config.Settings,
+		config:      config,
 	}, nil
 }
 
 // Load the given input and generates the input based on it
 func (t *Template) Load(file string) (common.MapStr, error) {
+
 	fields, err := common.LoadFieldsYaml(file)
 	if err != nil {
 		return nil, err
 	}
 
+	if len(t.config.AppendFields) > 0 {
+		cfgwarn.Experimental("append_fields is used.")
+		fields, err = appendFields(fields, t.config.AppendFields)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Start processing at the root
 	properties := common.MapStr{}
 	processor := Processor{EsVersion: t.esVersion}
@@ -155,7 +165,7 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 		indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
 	}
 
-	indexSettings.DeepUpdate(t.settings.Index)
+	indexSettings.DeepUpdate(t.config.Settings.Index)
 
 	var mappingName string
 	if t.esVersion.Major >= 6 {
@@ -182,9 +192,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 		},
 	}
 
-	if len(t.settings.Source) > 0 {
+	if len(t.config.Settings.Source) > 0 {
 		key := fmt.Sprintf("mappings.%s._source", mappingName)
-		basicStructure.Put(key, t.settings.Source)
+		basicStructure.Put(key, t.config.Settings.Source)
 	}
 
 	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
@@ -200,3 +210,19 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 
 	return basicStructure
 }
+
+func appendFields(fields, appendFields common.Fields) (common.Fields, error) {
+	if len(appendFields) > 0 {
+		appendFieldKeys := appendFields.GetKeys()
+
+		// Append is only allowed to add fields, not overwrite
+		for _, key := range appendFieldKeys {
+			if fields.HasNode(key) {
+				return nil, fmt.Errorf("append_fields contains an already existing key: %s", key)
+			}
+		}
+		// Appends fields to existing fields
+		fields = append(fields, appendFields...)
+	}
+	return fields, nil
+}

libbeat/template/template_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/common"
 )
 
 func TestNumberOfRoutingShards(t *testing.T) {
@@ -54,3 +56,106 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
 
 	assert.Equal(t, 5, shards.(int))
 }
+
+func TestAppendFields(t *testing.T) {
+	tests := []struct {
+		fields       common.Fields
+		appendFields common.Fields
+		error        bool
+	}{
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "b",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: false,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "b",
+						},
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: true,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: true,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+				},
+			},
+			error: true,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := appendFields(test.fields, test.appendFields)
+		if test.error {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}