Skip to content

Commit

Permalink
Fix index template loading when the new index is selected (elastic#22682
Browse files Browse the repository at this point in the history
)

## What does this PR do?

This PR fixes the `setup.template.type` setting. When `index` were selected, Beats based the generated index on the legacy format, not on the component. This prevented Beats from loading the index.

## Why is it important?

`setup.template.type=index` was broken. Now it is fixed.

## How to test this PR locally

```
./filebeat setup --index-management -E setup.template.type=index
```

## Related issues

Closes elastic#22482

(cherry picked from commit 569268d)
  • Loading branch information
kvch committed Nov 24, 2020
1 parent 53a2e7e commit fbb7fb6
Show file tree
Hide file tree
Showing 6 changed files with 59 additions and 13 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851]
- Fix parsing of expired licences. {issue}21112[21112] {pull}22180[22180]
- Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438]
- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682]

*Auditbeat*

Expand Down
8 changes: 4 additions & 4 deletions libbeat/template/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,8 @@ func (t *Template) loadMinimalComponent() common.MapStr {
}

func (t *Template) loadMinimalIndex() common.MapStr {
m := t.loadMinimalLegacy()
m := t.loadMinimalComponent()
m["priority"] = t.priority
delete(m, "order")
return m
}

Expand Down Expand Up @@ -304,9 +303,10 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr {
}

func (t *Template) generateIndex(properties common.MapStr) common.MapStr {
tmpl := t.generateLegacy(properties)
tmpl := t.generateComponent(properties)
tmpl["priority"] = t.priority
delete(tmpl, "order")
keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern())
tmpl[keyPattern] = patterns
return tmpl
}

Expand Down
19 changes: 18 additions & 1 deletion libbeat/tests/system/idxmgmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,28 @@ def assert_index_template_not_loaded(self, template):
with pytest.raises(NotFoundError):
self._client.transport.perform_request('GET', '/_template/' + template)

def assert_index_template_loaded(self, template):
def assert_legacy_index_template_loaded(self, template):
resp = self._client.transport.perform_request('GET', '/_template/' + template)
assert template in resp
assert "lifecycle" not in resp[template]["settings"]["index"]

def assert_index_template_loaded(self, template):
resp = self._client.transport.perform_request('GET', '/_index_template/' + template)
found = False
for index_template in resp['index_templates']:
if index_template['name'] == template:
found = True
assert found

def assert_component_template_loaded(self, template):
resp = self._client.transport.perform_request('GET', '/_component_template/' + template)
found = False
print(resp)
for index_template in resp['component_templates']:
if index_template['name'] == template:
found = True
assert found

def assert_ilm_template_loaded(self, template, policy, alias):
resp = self._client.transport.perform_request('GET', '/_template/' + template)
assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy
Expand Down
8 changes: 4 additions & 4 deletions libbeat/tests/system/test_cmd_setup_index_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ def test_setup_ilm_disabled(self):
"-E", "setup.ilm.enabled=false"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)

Expand Down Expand Up @@ -242,7 +242,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self):
"-E", "setup.template.pattern=" + self.custom_template + "*"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template)
self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"])
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
Expand All @@ -261,7 +261,7 @@ def test_setup_template_with_opts(self):
"-E", "setup.template.settings.index.number_of_shards=2"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)

# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
Expand All @@ -284,7 +284,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self):
"-E", "setup.template.name=" + self.custom_alias,
"-E", "setup.template.pattern=" + self.custom_alias + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.policy_name)

# ensure ilm policy is created, triggering overwriting existing template
Expand Down
4 changes: 2 additions & 2 deletions libbeat/tests/system/test_ilm.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def test_ilm_disabled(self):
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()

self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)

Expand Down Expand Up @@ -234,7 +234,7 @@ def test_setup_ilm_disabled(self):
"-E", "setup.ilm.enabled=false"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)

Expand Down
32 changes: 30 additions & 2 deletions libbeat/tests/system/test_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ def test_setup_template_with_opts(self):
"-E", "setup.template.settings.index.number_of_shards=2"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)

# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
Expand Down Expand Up @@ -305,7 +305,7 @@ def test_template_created_on_ilm_policy_created(self):
"-E", "setup.template.name=" + self.custom_alias,
"-E", "setup.template.pattern=" + self.custom_alias + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.policy_name)

# ensure ilm policy is created, triggering overwriting existing template
Expand All @@ -322,6 +322,34 @@ def test_template_created_on_ilm_policy_created(self):
index = resp[self.custom_alias]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]

@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_index(self):
"""
Test template setup of new index templates
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.template.type=index"])

assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)

@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_component(self):
"""
Test template setup of component index templates
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.template.type=component"])

assert exit_code == 0
self.idxmgmt.assert_component_template_loaded(self.index_name)


class TestCommandExportTemplate(BaseTest):
"""
Expand Down

0 comments on commit fbb7fb6

Please sign in to comment.