From 9efc4820f001fa862a00131a94ec986ec3c08954 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 23 Jul 2020 17:55:07 -0400 Subject: [PATCH 1/2] Increase index.max_docvalue_fields_search to 200 (#20218) The number of docvalue fields in Filebeat went beyond 100 and Discover was not loading. I added settings.index.max_docvalue_fields_search=200 to the default index template. In Filebeat there are about 117 fields now. Fixes #20215 (cherry picked from commit e95a9058c8389c5526fd74ad0880dff171fc0745) --- CHANGELOG.next.asciidoc | 2 ++ libbeat/template/template.go | 11 ++++++++--- libbeat/template/template_test.go | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5bcdb19cf8d..3fd41d1d72d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -417,6 +417,8 @@ field. You can revert this change by configuring tags for the module and omittin - Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464] - Add support to trim captured values in the dissect processor. {pull}19464[19464] - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] +- Add support for DNS over TLS for the dns_processor. {pull}19321[19321] +- Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] *Auditbeat* diff --git a/libbeat/template/template.go b/libbeat/template/template.go index b11599eb205..dac3a920196 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -32,9 +32,10 @@ import ( var ( // Defaults used in the template - defaultDateDetection = false - defaultTotalFieldsLimit = 10000 - defaultNumberOfRoutingShards = 30 + defaultDateDetection = false + defaultTotalFieldsLimit = 10000 + defaultNumberOfRoutingShards = 30 + defaultMaxDocvalueFieldsSearch = 200 // Array to store dynamicTemplate parts in dynamicTemplates []common.MapStr @@ -325,6 +326,10 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map indexSettings.Put("query.default_field", fields) } + if ver.Major >= 6 { + indexSettings.Put("max_docvalue_fields_search", defaultMaxDocvalueFieldsSearch) + } + indexSettings.DeepUpdate(userSettings) return indexSettings } diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 7e6a688db5d..52080274dd6 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -113,6 +113,7 @@ func TestTemplate(t *testing.T) { template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) template.Assert("order", 1) template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + template.Assert("settings.index.max_docvalue_fields_search", 200) }) t.Run("for ES 7.x", func(t *testing.T) { @@ -120,6 +121,7 @@ func TestTemplate(t *testing.T) { template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) template.Assert("order", 1) template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + template.Assert("settings.index.max_docvalue_fields_search", 200) }) t.Run("for ES 8.x", func(t *testing.T) { @@ -127,6 +129,7 @@ func TestTemplate(t *testing.T) { template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) template.Assert("order", 1) template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + template.Assert("settings.index.max_docvalue_fields_search", 200) }) } From 6f05f85ebf5a6a0bca057add95798461d5ce0378 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 28 Jul 2020 16:12:00 -0400 Subject: [PATCH 2/2] Remove extra line --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3fd41d1d72d..9cf7ee29a46 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -417,7 +417,6 @@ field. You can revert this change by configuring tags for the module and omittin - Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464] - Add support to trim captured values in the dissect processor. {pull}19464[19464] - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] -- Add support for DNS over TLS for the dns_processor. {pull}19321[19321] - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] *Auditbeat*