From adaef57db326865a68bef34e2892dd513892adf7 Mon Sep 17 00:00:00 2001
From: Andrew Kroh <andrew.kroh@elastic.co>
Date: Thu, 23 Jul 2020 17:55:07 -0400
Subject: [PATCH] Increase index.max_docvalue_fields_search to 200 (#20218)

The number of docvalue fields in Filebeat went beyond 100 and Discover was not loading.
I added settings.index.max_docvalue_fields_search=200 to the default index template.
In Filebeat there are about 117 fields now.

Fixes #20215

(cherry picked from commit e95a9058c8389c5526fd74ad0880dff171fc0745)
---
 CHANGELOG.next.asciidoc           | 13 +++++++++++++
 libbeat/template/template.go      | 11 ++++++++---
 libbeat/template/template_test.go |  3 +++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index d1ef5d1b0d1..74bc9a2bf3c 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -197,6 +197,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808]
+- Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
+- Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905]
+- Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
+- Add support for multiple sets of hints on autodiscover {pull}18883[18883]
+- Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181]
+- Add data type conversion in `dissect` processor for converting string values to other basic data types. {pull}18683[18683]
+- Add the `ignore_failure` configuration option to the dissect processor. {pull}19464[19464]
+- Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464]
+- Add support to trim captured values in the dissect processor. {pull}19464[19464]
+- Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
+- Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
+- Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
 
 *Auditbeat*
 
diff --git a/libbeat/template/template.go b/libbeat/template/template.go
index b11599eb205..dac3a920196 100644
--- a/libbeat/template/template.go
+++ b/libbeat/template/template.go
@@ -32,9 +32,10 @@ import (
 
 var (
 	// Defaults used in the template
-	defaultDateDetection         = false
-	defaultTotalFieldsLimit      = 10000
-	defaultNumberOfRoutingShards = 30
+	defaultDateDetection           = false
+	defaultTotalFieldsLimit        = 10000
+	defaultNumberOfRoutingShards   = 30
+	defaultMaxDocvalueFieldsSearch = 200
 
 	// Array to store dynamicTemplate parts in
 	dynamicTemplates []common.MapStr
@@ -325,6 +326,10 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map
 		indexSettings.Put("query.default_field", fields)
 	}
 
+	if ver.Major >= 6 {
+		indexSettings.Put("max_docvalue_fields_search", defaultMaxDocvalueFieldsSearch)
+	}
+
 	indexSettings.DeepUpdate(userSettings)
 	return indexSettings
 }
diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go
index 7e6a688db5d..52080274dd6 100644
--- a/libbeat/template/template_test.go
+++ b/libbeat/template/template_test.go
@@ -113,6 +113,7 @@ func TestTemplate(t *testing.T) {
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
 		template.Assert("order", 1)
 		template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+		template.Assert("settings.index.max_docvalue_fields_search", 200)
 	})
 
 	t.Run("for ES 7.x", func(t *testing.T) {
@@ -120,6 +121,7 @@ func TestTemplate(t *testing.T) {
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
 		template.Assert("order", 1)
 		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+		template.Assert("settings.index.max_docvalue_fields_search", 200)
 	})
 
 	t.Run("for ES 8.x", func(t *testing.T) {
@@ -127,6 +129,7 @@ func TestTemplate(t *testing.T) {
 		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
 		template.Assert("order", 1)
 		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+		template.Assert("settings.index.max_docvalue_fields_search", 200)
 	})
 }