Add support for index to fields.yml (elastic#4853)

Setting `index: false` for a field was not possible so far. This adds this possibility. For more details in `index` see: https://www.elastic.co/guide/en/elasticsearch/reference/5.5/mapping-index.html

Additional changes:

* `enable` and `dynamic` are only available for objects. The functionality was move to object. https://www.elastic.co/guide/en/elasticsearch/reference/5.5/enabled.html
* Add support for `enable` and `index` to the docs script so it is visible in the docs which fields are indexed and which ones not

libbeat/scripts/generate_fields_docs.py

@@ -54,6 +54,14 @@ def document_field(output, field, path):
     if "description" in field:
         output.write("{}\n\n".format(field["description"]))
 
+    if "index" in field:
+        if not field["index"]:
+            output.write("{}\n\n".format("Field is not indexed."))
+
+    if "enable" in field:
+        if not field["enable"]:
+            output.write("{}\n\n".format("Object is not enabled."))
+
 
 def fields_to_asciidoc(input, output, beat):
 

libbeat/template/field.go

@@ -25,6 +25,7 @@ type Field struct {
 	SearchAnalyzer string      `config:"search_analyzer"`
 	Norms          bool        `config:"norms"`
 	Dynamic        dynamicType `config:"dynamic"`
+	Index          *bool       `config:"index"`
 
 	path      string
 	esVersion common.Version
@@ -161,6 +162,14 @@ func (f *Field) object() common.MapStr {
 
 	properties := f.getDefaultProperties()
 	properties["type"] = "object"
+	if f.Enabled != nil {
+		properties["enabled"] = *f.Enabled
+	}
+
+	if f.Dynamic.value != nil {
+		properties["dynamic"] = f.Dynamic.value
+	}
+
 	return properties
 }
 
@@ -183,16 +192,13 @@ func (f *Field) addDynamicTemplate(properties common.MapStr, matchType string) {
 
 func (f *Field) getDefaultProperties() common.MapStr {
 	// Currently no defaults exist
-	property := common.MapStr{}
-	if f.Enabled != nil {
-		property["enabled"] = *f.Enabled
-	}
+	properties := common.MapStr{}
 
-	if f.Dynamic.value != nil {
-		property["dynamic"] = f.Dynamic.value
+	if f.Index != nil {
+		properties["index"] = *f.Index
 	}
 
-	return property
+	return properties
 }
 
 // Recursively generates the correct key based on the dots

libbeat/template/field_test.go

@@ -14,6 +14,7 @@ func TestField(t *testing.T) {
 	assert.NoError(t, err)
 
 	falseVar := false
+	trueVar := true
 
 	tests := []struct {
 		field  Field
@@ -52,7 +53,7 @@ func TestField(t *testing.T) {
 		},
 		{
 			field:  Field{Type: "object", Enabled: &falseVar},
-			method: func(f Field) common.MapStr { return f.other() },
+			method: func(f Field) common.MapStr { return f.object() },
 			output: common.MapStr{
 				"type":    "object",
 				"enabled": false,
@@ -135,23 +136,37 @@ func TestField(t *testing.T) {
 		},
 		{
 			field:  Field{Dynamic: dynamicType{false}},
-			method: func(f Field) common.MapStr { return f.other() },
+			method: func(f Field) common.MapStr { return f.object() },
 			output: common.MapStr{
-				"dynamic": false,
+				"dynamic": false, "type": "object",
 			},
 		},
 		{
 			field:  Field{Dynamic: dynamicType{true}},
-			method: func(f Field) common.MapStr { return f.other() },
+			method: func(f Field) common.MapStr { return f.object() },
 			output: common.MapStr{
-				"dynamic": true,
+				"dynamic": true, "type": "object",
 			},
 		},
 		{
 			field:  Field{Dynamic: dynamicType{"strict"}},
+			method: func(f Field) common.MapStr { return f.object() },
+			output: common.MapStr{
+				"dynamic": "strict", "type": "object",
+			},
+		},
+		{
+			field:  Field{Type: "long", Index: &falseVar},
+			method: func(f Field) common.MapStr { return f.other() },
+			output: common.MapStr{
+				"type": "long", "index": false,
+			},
+		},
+		{
+			field:  Field{Type: "text", Index: &trueVar},
 			method: func(f Field) common.MapStr { return f.other() },
 			output: common.MapStr{
-				"dynamic": "strict",
+				"type": "text", "index": true,
 			},
 		},
 	}