From fbb7fb65941940f1432caf5d341c27c19f68ac61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 24 Nov 2020 14:55:03 +0100 Subject: [PATCH] Fix index template loading when the new index is selected (#22682) ## What does this PR do? This PR fixes the `setup.template.type` setting. When `index` were selected, Beats based the generated index on the legacy format, not on the component. This prevented Beats from loading the index. ## Why is it important? `setup.template.type=index` was broken. Now it is fixed. ## How to test this PR locally ``` ./filebeat setup --index-management -E setup.template.type=index ``` ## Related issues Closes #22482 (cherry picked from commit 569268dd5989fe06d09b4a8fd9e9b81d69685b3c) --- CHANGELOG.next.asciidoc | 1 + libbeat/template/template.go | 8 ++--- libbeat/tests/system/idxmgmt.py | 19 ++++++++++- .../system/test_cmd_setup_index_management.py | 8 ++--- libbeat/tests/system/test_ilm.py | 4 +-- libbeat/tests/system/test_template.py | 32 +++++++++++++++++-- 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 55ff84b39f9..8abd7aafd80 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -157,6 +157,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851] - Fix parsing of expired licences. {issue}21112[21112] {pull}22180[22180] - Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438] +- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682] *Auditbeat* diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 0fbfa30ccab..f6366ddc676 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -235,9 +235,8 @@ func (t *Template) loadMinimalComponent() common.MapStr { } func (t *Template) loadMinimalIndex() common.MapStr { - m := t.loadMinimalLegacy() + m := t.loadMinimalComponent() m["priority"] = t.priority - delete(m, "order") return m } @@ -304,9 +303,10 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr { } func (t *Template) generateIndex(properties common.MapStr) common.MapStr { - tmpl := t.generateLegacy(properties) + tmpl := t.generateComponent(properties) tmpl["priority"] = t.priority - delete(tmpl, "order") + keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) + tmpl[keyPattern] = patterns return tmpl } diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 864d134a9c9..f789b919bc2 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -56,11 +56,28 @@ def assert_index_template_not_loaded(self, template): with pytest.raises(NotFoundError): self._client.transport.perform_request('GET', '/_template/' + template) - def assert_index_template_loaded(self, template): + def assert_legacy_index_template_loaded(self, template): resp = self._client.transport.perform_request('GET', '/_template/' + template) assert template in resp assert "lifecycle" not in resp[template]["settings"]["index"] + def assert_index_template_loaded(self, template): + resp = self._client.transport.perform_request('GET', '/_index_template/' + template) + found = False + for index_template in resp['index_templates']: + if index_template['name'] == template: + found = True + assert found + + def assert_component_template_loaded(self, template): + resp = self._client.transport.perform_request('GET', '/_component_template/' + template) + found = False + print(resp) + for index_template in resp['component_templates']: + if index_template['name'] == template: + found = True + assert found + def assert_ilm_template_loaded(self, template, policy, alias): resp = self._client.transport.perform_request('GET', '/_template/' + template) assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 234f4e76a0b..cd6d19eefbe 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -110,7 +110,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -242,7 +242,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): "-E", "setup.template.pattern=" + self.custom_template + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_template) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template) self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"]) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -261,7 +261,7 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) # check that settings are overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) @@ -284,7 +284,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): "-E", "setup.template.name=" + self.custom_alias, "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 3d37125f6e4..2913893e11c 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -68,7 +68,7 @@ def test_ilm_disabled(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @@ -234,7 +234,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 67a34457484..35dc8915a08 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -267,7 +267,7 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) # check that settings are overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) @@ -305,7 +305,7 @@ def test_template_created_on_ilm_policy_created(self): "-E", "setup.template.name=" + self.custom_alias, "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template @@ -322,6 +322,34 @@ def test_template_created_on_ilm_policy_created(self): index = resp[self.custom_alias]["settings"]["index"] assert index["number_of_shards"] == "2", index["number_of_shards"] + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.tag('integration') + def test_setup_template_index(self): + """ + Test template setup of new index templates + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.setupCmd, + "-E", "setup.template.type=index"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.tag('integration') + def test_setup_template_component(self): + """ + Test template setup of component index templates + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.setupCmd, + "-E", "setup.template.type=component"]) + + assert exit_code == 0 + self.idxmgmt.assert_component_template_loaded(self.index_name) + class TestCommandExportTemplate(BaseTest): """