From 028a3776eee926a21e11ac5de1e7ed464944f409 Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 7 May 2019 20:00:47 +0200 Subject: [PATCH 1/7] Add and deprecate setup cmds for index handling Deprecate `setup --template` and `setup --ilm` in favour of newly introduced `setup --index-management` command. implements #12095 --- libbeat/cmd/instance/beat.go | 51 ++-- libbeat/cmd/setup.go | 29 ++- libbeat/idxmgmt/componentname_string.go | 41 ++++ libbeat/idxmgmt/idxmgmt.go | 4 +- libbeat/idxmgmt/std.go | 122 +++++++--- .../idxmgmt/{idxmgmt_test.go => std_test.go} | 94 ++++++-- .../system/test_cmd_setup_index_management.py | 220 ++++++++++++++++++ libbeat/tests/system/test_ilm.py | 42 +++- libbeat/tests/system/test_template.py | 65 +++++- 9 files changed, 569 insertions(+), 99 deletions(-) create mode 100644 libbeat/idxmgmt/componentname_string.go rename libbeat/idxmgmt/{idxmgmt_test.go => std_test.go} (81%) create mode 100644 libbeat/tests/system/test_cmd_setup_index_management.py diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 78ea2ab62fa..77436a6cfc9 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error { //SetupSettings holds settings necessary for beat setup type SetupSettings struct { - Template bool Dashboard bool MachineLearning bool Pipeline bool - ILMPolicy bool + IndexManagement bool + //Deprecated: use IndexManagementKey instead + Template bool + //Deprecated: use IndexManagementKey instead + ILMPolicy bool } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. @@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return err } - if setup.Template || setup.ILMPolicy { + if setup.IndexManagement || setup.Template || setup.ILMPolicy { outCfg := b.Config.Output - if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } + esClient, err := elasticsearch.NewConnectedClient(outCfg.Config()) + if err != nil { + return err + } - esConfig := outCfg.Config() - if b.IdxSupporter.Enabled() { - esClient, err := elasticsearch.NewConnectedClient(esConfig) - if err != nil { - return err - } - - // prepare index by loading templates, lifecycle policies and write aliases - - m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) - var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset - if setup.Template { - tmplLoadMode = idxmgmt.LoadModeOverwrite - } - if setup.ILMPolicy { - ilmLoadMode = idxmgmt.LoadModeOverwrite - } - - err = m.Setup(tmplLoadMode, ilmLoadMode) - if err != nil { - return err - } + var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset + if setup.IndexManagement || setup.Template { + loadTemplate = idxmgmt.LoadModeOverwrite + } + if setup.IndexManagement || setup.ILMPolicy { + loadILM = idxmgmt.LoadModeOverwrite + } + m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) + if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok { + fmt.Println(warn) + } + if err = m.Setup(loadTemplate, loadILM); err != nil { + return err } - fmt.Println("Index setup complete.") + fmt.Println("Index setup finished.") } if setup.Dashboard { diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index eeed4389e8f..f3ed31b7afb 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -28,15 +28,23 @@ import ( ) const ( - //TemplateKey used for defining template in setup cmd - TemplateKey = "template" //DashboardKey used for registering dashboards in setup cmd DashboardKey = "dashboards" //MachineLearningKey used for registering ml jobs in setup cmd MachineLearningKey = "machine-learning" //PipelineKey used for registering pipelines in setup cmd PipelineKey = "pipelines" - //ILMPolicyKey used for registering ilm in setup cmd + //IndexManagementKey used for loading all components related to ES index management in setup cmd + IndexManagementKey = "index-management" + + //TemplateKey used for loading template in setup cmd + // + //Deprecated: use IndexManagementKey instead + TemplateKey = "template" + + //ILMPolicyKey used for loading ilm in setup cmd + // + //Deprecated: use IndexManagementKey instead ILMPolicyKey = "ilm-policy" ) @@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co } var registeredFlags = map[string]bool{ - TemplateKey: false, DashboardKey: false, MachineLearningKey: false, PipelineKey: false, + IndexManagementKey: false, + TemplateKey: false, ILMPolicyKey: false, } var setupAll = true @@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co for k, v := range registeredFlags { if setupAll || v { switch k { - case TemplateKey: - s.Template = true case DashboardKey: s.Dashboard = true case MachineLearningKey: s.MachineLearning = true case PipelineKey: s.Pipeline = true + case IndexManagementKey: + s.IndexManagement = true case ILMPolicyKey: s.ILMPolicy = true + case TemplateKey: + s.Template = true } } } @@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co }, } - setup.Flags().Bool(TemplateKey, false, "Setup index template") setup.Flags().Bool(DashboardKey, false, "Setup dashboards") setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations") setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines") + setup.Flags().Bool(IndexManagementKey, false, + "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias") + setup.Flags().Bool(TemplateKey, false, "Setup index template") + setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey)) setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy") + setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey)) return &setup } diff --git a/libbeat/idxmgmt/componentname_string.go b/libbeat/idxmgmt/componentname_string.go new file mode 100644 index 00000000000..23f07f5daa6 --- /dev/null +++ b/libbeat/idxmgmt/componentname_string.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by "stringer -linecomment -type componentName"; DO NOT EDIT. + +package idxmgmt + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[componentTemplate-0] + _ = x[componentILM-1] +} + +const _componentName_name = "templateilm" + +var _componentName_index = [...]uint8{0, 8, 11} + +func (i componentName) String() string { + if i >= componentName(len(_componentName_index)-1) { + return "componentName(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _componentName_name[_componentName_index[i]:_componentName_index[i+1]] +} diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go index 6ec75b033ef..a18eae5d8ff 100644 --- a/libbeat/idxmgmt/idxmgmt.go +++ b/libbeat/idxmgmt/idxmgmt.go @@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er // A manager instantiated via Supporter is responsible for instantiating/configuring // the index throughout the Elastic Stack. type Supporter interface { - // Enalbed checks if index management is configured to configure templates, - // ILM, or aliases. + // Enabled checks if index management is configured to setup templates or ILM Enabled() bool // BuildSelector create an index selector. @@ -62,6 +61,7 @@ type Asseter interface { // Manager is used to initialize indices, ILM policies, and aliases within the // Elastic Stack. type Manager interface { + VerifySetup(template, ilm LoadMode) (bool, string) Setup(template, ilm LoadMode) error } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index ea655bb71a8..3c168c2dbd6 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -62,6 +62,33 @@ type ilmIndexSelector struct { st *indexState } +type componentName uint8 + +//go:generate stringer -linecomment -type componentName +const ( + componentTemplate componentName = iota //template + componentILM //ilm +) + +type component struct { + name componentName + enabled, overwrite, load bool +} + +func newComponent(name componentName, enabled, overwrite bool, mode LoadMode) component { + if mode == LoadModeUnset && !enabled { + mode = LoadModeDisabled + } + if mode >= LoadModeOverwrite { + overwrite = true + } + if mode == LoadModeForce { + enabled = true + } + load := mode.Enabled() && enabled + return component{name: name, enabled: enabled, overwrite: overwrite, load: load} +} + func newIndexSupport( log *logp.Logger, info beat.Info, @@ -95,7 +122,17 @@ func newIndexSupport( } func (s *indexSupport) Enabled() bool { - return s.templateCfg.Enabled || (s.ilm.Mode() != ilm.ModeDisabled) + return s.enabled(componentTemplate) || s.enabled(componentILM) +} + +func (s *indexSupport) enabled(c componentName) bool { + switch c { + case componentTemplate: + return s.templateCfg.Enabled + case componentILM: + return s.ilm.Mode() != ilm.ModeDisabled + } + return false } func (s *indexSupport) Manager( @@ -176,41 +213,55 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, st: &s.st, }, nil } +func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { + ilmComponent := newComponent(componentILM, m.support.enabled(componentILM), false, loadILM) + templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate), + m.support.templateCfg.Overwrite, loadTemplate) + + if ilmComponent.load && !templateComponent.load { + return false, "Loading ILM policy and write alias without loading template " + + "is not recommended. Check your configuration." + } + if templateComponent.load && !ilmComponent.load && ilmComponent.enabled { + return false, "Loading template with ILM settings whithout loading ILM policy and alias can lead " + + "to issues and is not recommended. Check your configuration" + } + var warn string + if !ilmComponent.load { + warn += "ILM policy and write alias loading not enabled. " + } + if !templateComponent.load { + warn += "Template loading not enabled." + } + return warn == "", warn +} func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { - var err error log := m.support.log - withILM := m.support.st.withILM.Load() - if !withILM { - withILM, err = m.ilm.Enabled() - if err != nil { - return err - } + withILM, err := m.setupWithILM() + if err != nil { + return err } - if loadILM == LoadModeUnset { - if withILM { - loadILM = LoadModeEnabled - log.Info("Auto ILM enable success.") - } else { - loadILM = LoadModeDisabled - } + if withILM && loadILM.Enabled() { + log.Info("Auto ILM enable success.") } - if loadILM == LoadModeForce || withILM && loadILM.Enabled() { - // mark ILM as enabled in indexState if withILM is true - m.support.st.withILM.CAS(false, true) + ilmComponent := newComponent(componentILM, withILM, false, loadILM) + templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate), + m.support.templateCfg.Overwrite, loadTemplate) + if ilmComponent.load { // install ilm policy - policyCreated, err := m.ilm.EnsurePolicy(loadILM >= LoadModeOverwrite) + policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite) if err != nil { return err } log.Info("ILM policy successfully loaded.") // The template should be updated if a new policy is created. - if policyCreated && loadTemplate.Enabled() { - loadTemplate = LoadModeOverwrite + if policyCreated && templateComponent.enabled { + templateComponent.overwrite = true } // create alias @@ -224,24 +275,17 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { } } - // create and install template - if loadTemplate == LoadModeForce || m.support.templateCfg.Enabled && loadTemplate.Enabled() { + if templateComponent.load { tmplCfg := m.support.templateCfg + tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled - if withILM { + if ilmComponent.enabled { ilm := m.support.ilm tmplCfg, err = applyILMSettings(log, tmplCfg, ilm.Policy(), ilm.Alias()) if err != nil { return err } } - - if loadTemplate == LoadModeForce { - tmplCfg.Enabled = true - } - if loadTemplate >= LoadModeOverwrite { - tmplCfg.Overwrite = true - } fields := m.assets.Fields(m.support.info.Beat) err = m.clientHandler.Load(tmplCfg, m.support.info, fields, m.support.migration) if err != nil { @@ -254,6 +298,22 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { return nil } +func (m *indexManager) setupWithILM() (bool, error) { + var err error + withILM := m.support.st.withILM.Load() + if !withILM { + withILM, err = m.ilm.Enabled() + if err != nil { + return false, err + } + if withILM { + // mark ILM as enabled in indexState + m.support.st.withILM.CAS(false, true) + } + } + return withILM, nil +} + func (s *ilmIndexSelector) Select(evt *beat.Event) (string, error) { if idx := getEventCustomIndex(evt); idx != "" { return idx, nil diff --git a/libbeat/idxmgmt/idxmgmt_test.go b/libbeat/idxmgmt/std_test.go similarity index 81% rename from libbeat/idxmgmt/idxmgmt_test.go rename to libbeat/idxmgmt/std_test.go index 01705dfa7a1..b5b9c44eafb 100644 --- a/libbeat/idxmgmt/idxmgmt_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -36,10 +36,10 @@ func TestDefaultSupport_Enabled(t *testing.T) { cases := map[string]struct { ilmCalls []onCall cfg map[string]interface{} - want bool + enabled bool }{ "templates and ilm disabled": { - want: false, + enabled: false, ilmCalls: []onCall{ onMode().Return(ilm.ModeDisabled), }, @@ -48,7 +48,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "templates only": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeDisabled), }, @@ -57,7 +57,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "ilm only": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeEnabled), }, @@ -66,7 +66,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "ilm tentatively": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeAuto), }, @@ -81,7 +81,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { factory := MakeDefaultSupport(makeMockILMSupport(test.ilmCalls...)) im, err := factory(nil, info, common.MustNewConfigFrom(test.cfg)) require.NoError(t, err) - assert.Equal(t, test.want, im.Enabled()) + assert.Equal(t, test.enabled, im.Enabled()) }) } } @@ -197,7 +197,59 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { } } -func TestDefaultSupport_TemplateHandling(t *testing.T) { +func TestIndexManager_VerifySetup(t *testing.T) { + for name, setup := range map[string]struct { + tmpl, ilm bool + loadTmpl, loadILM LoadMode + ok bool + warn string + }{ + "load template with ilm without loading ilm": { + ilm: true, tmpl: true, loadILM: LoadModeDisabled, + warn: "whithout loading ILM policy and alias", + }, + "load ilm without template": { + ilm: true, loadILM: LoadModeUnset, + warn: "without loading template is not recommended", + }, + "template disabled but loading enabled": { + loadTmpl: LoadModeEnabled, + warn: "loading not enabled", + }, + "ilm disabled but loading enabled": { + loadILM: LoadModeEnabled, tmpl: true, + warn: "loading not enabled", + }, + "ilm enabled but loading disabled": { + ilm: true, loadILM: LoadModeDisabled, + warn: "loading not enabled", + }, + "template enabled but loading disabled": { + tmpl: true, loadTmpl: LoadModeDisabled, + warn: "loading not enabled", + }, + "everything enabled": { + tmpl: true, loadTmpl: LoadModeUnset, ilm: true, loadILM: LoadModeUnset, + ok: true, + }, + } { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigFrom(common.MapStr{ + "setup.ilm.enabled": setup.ilm, + "setup.template.enabled": setup.tmpl, + }) + require.NoError(t, err) + support, err := MakeDefaultSupport(ilm.StdSupport)(nil, beat.Info{}, cfg) + require.NoError(t, err) + manager := support.Manager(newMockClientHandler(), nil) + ok, warn := manager.VerifySetup(setup.loadTmpl, setup.loadILM) + assert.Equal(t, setup.ok, ok) + assert.Contains(t, warn, setup.warn) + }) + } +} + +func TestIndexManager_Setup(t *testing.T) { cloneCfg := func(c template.TemplateConfig) template.TemplateConfig { if c.AppendFields != nil { tmp := make(mapping.Fields, len(c.AppendFields)) @@ -235,7 +287,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { tmplCfg *template.TemplateConfig alias, policy string }{ - "template default, ilm default": { + "template default ilm default": { tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ "overwrite": "true", "name": "test-9.9.9", @@ -246,7 +298,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "test-9.9.9", policy: "test-9.9.9", }, - "template default, ilm default with alias and policy changed": { + "template default ilm default with alias and policy changed": { cfg: common.MapStr{ "setup.ilm.rollover_alias": "mocktest", "setup.ilm.policy_name": "policy-keep", @@ -261,14 +313,14 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "mocktest", policy: "policy-keep", }, - "template default, ilm disabled": { + "template default ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, loadTemplate: LoadModeEnabled, tmplCfg: &defaultCfg, }, - "template default loadMode Overwrite, ilm disabled": { + "template default loadMode Overwrite ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, @@ -277,7 +329,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "overwrite": "true", }), }, - "template default loadMode Force, ilm disabled": { + "template default loadMode Force ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, @@ -286,27 +338,27 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "overwrite": "true", }), }, - "template loadMode disabled, ilm disabled": { + "template loadMode disabled ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, loadTemplate: LoadModeDisabled, }, - "template disabled, ilm default": { + "template disabled ilm default": { cfg: common.MapStr{ "setup.template.enabled": false, }, alias: "test-9.9.9", policy: "test-9.9.9", }, - "template disabled, ilm disabled, loadMode Overwrite": { + "template disabled ilm disabled, loadMode Overwrite": { cfg: common.MapStr{ "setup.template.enabled": false, "setup.ilm.enabled": false, }, loadILM: LoadModeOverwrite, }, - "template disabled, ilm disabled, loadMode Force": { + "template disabled ilm disabled loadMode Force": { cfg: common.MapStr{ "setup.template.enabled": false, "setup.ilm.enabled": false, @@ -315,13 +367,13 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "test-9.9.9", policy: "test-9.9.9", }, - "template loadmode disabled, ilm loadMode enabled": { + "template loadmode disabled ilm loadMode enabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeEnabled, alias: "test-9.9.9", policy: "test-9.9.9", }, - "template default, ilm loadMode disabled": { + "template default ilm loadMode disabled": { loadILM: LoadModeDisabled, tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ "name": "test-9.9.9", @@ -330,7 +382,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), }, - "template loadmode disabled, ilm loadmode disabled": { + "template loadmode disabled ilm loadmode disabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeDisabled, }, @@ -350,9 +402,9 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { } else { require.NoError(t, err) if test.tmplCfg == nil { - assert.Nil(t, clientHandler.tl.tmplCfg) + require.Nil(t, clientHandler.tl.tmplCfg) } else { - assert.Equal(t, test.tmplCfg, clientHandler.tl.tmplCfg) + require.Equal(t, test.tmplCfg, clientHandler.tl.tmplCfg) } assert.Equal(t, test.alias, clientHandler.il.alias) assert.Equal(t, test.policy, clientHandler.il.policy) diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py new file mode 100644 index 00000000000..031d7db4700 --- /dev/null +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -0,0 +1,220 @@ +from base import BaseTest +from idxmgmt import IdxMgmt +import os +from nose.plugins.attrib import attr +import unittest +import shutil +import datetime +import logging + +INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) + + +class TestCommandSetupIndexManagement(BaseTest): + """ + Test beat command `setup` related to ILM policy + """ + + def setUp(self): + super(TestCommandSetupIndexManagement, self).setUp() + + self.cmd = "--index-management" + # auto-derived default settings, if nothing else is set + self.index_name = self.alias_name = self.policy_name = self.beat_name + "-9.9.9" + + self.custom_alias = self.beat_name + "_foo" + self.custom_policy = self.beat_name + "_bar" + self.custom_template = self.beat_name + "_foobar" + + self.es = self.es_client() + self.idxmgmt = IdxMgmt(self.es, self.index_name) + self.idxmgmt.delete(index=self.custom_policy) + self.idxmgmt.delete(index=self.custom_alias) + self.idxmgmt.delete(index=self.index_name) + + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("elasticsearch").setLevel(logging.ERROR) + + def tearDown(self): + self.idxmgmt.delete(index=self.custom_alias) + self.idxmgmt.delete(index=self.custom_policy) + self.idxmgmt.delete(index=self.index_name) + + def render_config(self, **kwargs): + self.render_config_template( + elasticsearch={"hosts": self.get_elasticsearch_url()}, + es_template_name=self.index_name, + **kwargs + ) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_default(self): + """ + Test setup --index-management with default config + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_policy_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_disabled(self): + """ + Test setup --index-management when ilm disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.template.enabled=false"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_not_loaded(self.index_name) + self.idxmgmt.assert_alias_created(self.index_name) + self.idxmgmt.assert_policy_created(self.index_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_ilm_disabled(self): + """ + Test setup --index-management when ilm disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_policy_name(self): + """ + Test setup --index-management when policy_name is configured + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.policy_name=" + self.custom_policy]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name) + self.idxmgmt.assert_policy_created(self.custom_policy) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_rollover_alias(self): + """ + Test setup --index-management when ilm.rollover_alias is configured + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) + self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.custom_alias) + self.idxmgmt.assert_alias_created(self.custom_alias) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_name_and_pattern(self): + """ + Test setup --index-management ignores template.name and template.pattern when ilm is enabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*"]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_name_and_pattern_on_ilm_disabled(self): + """ + Test setup --index-management respects template.name and template.pattern when ilm is disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.custom_template) + self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"]) + self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_with_opts(self): + """ + Test setup --index-management with config options + """ + self.render_config() + + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.settings.index.number_of_shards=2"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + + # check that settings are overwritten + resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) + assert self.index_name in resp + index = resp[self.index_name]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_overwrite_template_on_ilm_policy_created(self): + """ + Test setup --index-management overwrites template when new ilm policy is created + """ + + # ensure template with ilm rollover_alias name is created, but ilm policy not yet + self.render_config() + + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.name=" + self.custom_alias, + "-E", "setup.template.pattern=" + self.custom_alias + "*"]) + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_policy_not_created(self.custom_alias) + + # ensure ilm policy is created, triggering overwriting existing template + exit_code = self.run_beat(extra_args=["setup", self.cmd, + "-E", "setup.template.overwrite=false", + "-E", "setup.template.settings.index.number_of_shards=2", + "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias) + self.idxmgmt.assert_policy_created(self.index_name) + # check that template was overwritten + resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) + index = resp[self.custom_alias]["settings"]["index"], resp + assert index["number_of_shards"] == "2", index["number_of_shards"] diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 2dcc11edd6f..aede356fd79 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -6,6 +6,7 @@ import shutil import datetime import logging +import json INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -217,6 +218,7 @@ def test_setup_ilm_default(self): assert exit_code == 0 self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) self.idxmgmt.assert_docs_written_to_alias(self.alias_name) self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) @@ -321,7 +323,6 @@ def test_load_disabled(self): def test_changed_policy_name(self): """ Test ilm-policy export when policy name is changed - """ policy_name = "foo" @@ -331,3 +332,42 @@ def test_changed_policy_name(self): assert exit_code == 0 self.assert_log_contains_policy() self.assert_log_contains_write_alias() + + def test_export_to_file_absolute_path(self): + """ + Test export ilm policy to file with absolute file path + """ + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + exit_code = self.run_beat( + extra_args=["export", self.cmd, "--dir=" + base_path], + config=self.config) + + assert exit_code == 0 + + file = os.path.join(base_path, "policy", self.policy_name + '.json') + with open(file) as f: + policy = json.load(f) + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_size"] == "50gb", policy + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_age"] == "30d", policy + + os.remove(file) + + def test_export_to_file_relative_path(self): + """ + Test export ilm policy to file with relative file path + """ + path = os.path.join(os.path.dirname(__file__), "export") + exit_code = self.run_beat( + extra_args=["export", self.cmd, "--dir=" + path], + config=self.config) + + assert exit_code == 0 + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + file = os.path.join(base_path, "policy", self.policy_name + '.json') + with open(file) as f: + policy = json.load(f) + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_size"] == "50gb", policy + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_age"] == "30d", policy + + os.remove(file) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 962f5aed419..03bf017ee06 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -5,6 +5,7 @@ import unittest import shutil import logging +import json INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -88,17 +89,18 @@ def test_json_template(self): Test loading of json based template """ + template_name = "bla" + es = self.es_client() self.copy_files(["template.json"]) - path = os.path.join(self.working_dir, "template.json") - print path + self.render_config_template( elasticsearch={"hosts": self.get_host()}, template_overwrite="true", template_json_enabled="true", template_json_path=path, - template_json_name="bla", + template_json_name=template_name, ) proc = self.start_beat() @@ -107,8 +109,7 @@ def test_json_template(self): self.wait_until(lambda: self.log_contains("template with name 'bla' loaded")) proc.check_kill_and_wait() - es = self.es_client() - result = es.transport.perform_request('GET', '/_template/bla') + result = es.transport.perform_request('GET', '/_template/' + template_name) assert len(result) == 1 def get_host(self): @@ -335,9 +336,8 @@ def setUp(self): shutil.copy(os.path.join(self.beat_path, "fields.yml"), self.output) self.template_name = self.beat_name + "-9.9.9" - def assert_log_contains_template(self, template, index_pattern): + def assert_log_contains_template(self, index_pattern): assert self.log_contains('Loaded index template') - assert self.log_contains(template) assert self.log_contains(index_pattern) def test_default(self): @@ -351,7 +351,7 @@ def test_default(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, self.template_name + "-*") + self.assert_log_contains_template(self.template_name + "-*") def test_changed_index_pattern(self): """ @@ -367,7 +367,7 @@ def test_changed_index_pattern(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, alias_name + "-*") + self.assert_log_contains_template(alias_name + "-*") def test_load_disabled(self): """ @@ -380,4 +380,49 @@ def test_load_disabled(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, self.template_name + "-*") + self.assert_log_contains_template(self.template_name + "-*") + + def test_export_to_file_absolute_path(self): + """ + Test export template to file with absolute file path + """ + self.render_config_template(self.beat_name, self.output, + fields=self.output) + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + exit_code = self.run_beat( + extra_args=["export", "template", "--dir=" + base_path], + config=self.config) + + assert exit_code == 0 + + file = os.path.join(base_path, "template", self.template_name + '.json') + with open(file) as f: + template = json.load(f) + assert 'index_patterns' in template + assert template['index_patterns'] == [self.template_name + '-*'], template + + os.remove(file) + + def test_export_to_file_relative_path(self): + """ + Test export template to file with relative file path + """ + self.render_config_template(self.beat_name, self.output, + fields=self.output) + + path = os.path.join(os.path.dirname(__file__), "export") + exit_code = self.run_beat( + extra_args=["export", "template", "--dir=" + path], + config=self.config) + + assert exit_code == 0 + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + file = os.path.join(base_path, "template", self.template_name + '.json') + with open(file) as f: + template = json.load(f) + assert 'index_patterns' in template + assert template['index_patterns'] == [self.template_name + '-*'], template + + os.remove(file) From eb20f9d4022958a86aec7b2940d0c16f461ed0a2 Mon Sep 17 00:00:00 2001 From: simitt Date: Thu, 9 May 2019 15:09:15 +0200 Subject: [PATCH 2/7] Add changelog. --- CHANGELOG-developer.next.asciidoc | 1 + libbeat/tests/system/idxmgmt.py | 6 +++--- libbeat/tests/system/test_cmd_setup_index_management.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index c0a07bcd80e..d7d9ee5c38b 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -37,3 +37,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Jinja2 version to 2.10.1. {pull}11817[11817] - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067] - Update urllib3 version to 1.24.2 {pull}11930[11930] +- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132] diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index caaa7d7d974..0c673ccfb28 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -14,10 +14,10 @@ def __init__(self, client, index): def needs_init(self, s): return s == '' or s == '*' - def delete(self, index="", template="", policy=""): + def delete(self, index=""): self.delete_index_and_alias(index=index) - self.delete_template(template=template) - self.delete_policy(policy=policy) + self.delete_template(template=index) + self.delete_policy(policy=index) def delete_index_and_alias(self, index=""): if self.needs_init(index): diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 031d7db4700..8f556942adc 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -196,7 +196,6 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): # ensure template with ilm rollover_alias name is created, but ilm policy not yet self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], extra_args=["setup", self.cmd, "-E", "setup.ilm.enabled=false", @@ -204,7 +203,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): "-E", "setup.template.pattern=" + self.custom_alias + "*"]) assert exit_code == 0 self.idxmgmt.assert_index_template_loaded(self.custom_alias) - self.idxmgmt.assert_policy_not_created(self.custom_alias) + self.idxmgmt.assert_policy_not_created(self.index_name) # ensure ilm policy is created, triggering overwriting existing template exit_code = self.run_beat(extra_args=["setup", self.cmd, @@ -216,5 +215,6 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): self.idxmgmt.assert_policy_created(self.index_name) # check that template was overwritten resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) - index = resp[self.custom_alias]["settings"]["index"], resp + assert self.custom_alias in resp + index = resp[self.custom_alias]["settings"]["index"] assert index["number_of_shards"] == "2", index["number_of_shards"] From 63c739828a1904271863197cb2ed032c0de0c13b Mon Sep 17 00:00:00 2001 From: simitt Date: Thu, 9 May 2019 20:32:59 +0200 Subject: [PATCH 3/7] minor changes --- ...name_string.go => componenttype_string.go} | 14 +++---- libbeat/idxmgmt/std.go | 42 ++++++++++--------- 2 files changed, 29 insertions(+), 27 deletions(-) rename libbeat/idxmgmt/{componentname_string.go => componenttype_string.go} (73%) diff --git a/libbeat/idxmgmt/componentname_string.go b/libbeat/idxmgmt/componenttype_string.go similarity index 73% rename from libbeat/idxmgmt/componentname_string.go rename to libbeat/idxmgmt/componenttype_string.go index 23f07f5daa6..5718922aac9 100644 --- a/libbeat/idxmgmt/componentname_string.go +++ b/libbeat/idxmgmt/componenttype_string.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// Code generated by "stringer -linecomment -type componentName"; DO NOT EDIT. +// Code generated by "stringer -linecomment -type componentType"; DO NOT EDIT. package idxmgmt @@ -29,13 +29,13 @@ func _() { _ = x[componentILM-1] } -const _componentName_name = "templateilm" +const _componentType_name = "templateilm" -var _componentName_index = [...]uint8{0, 8, 11} +var _componentType_index = [...]uint8{0, 8, 11} -func (i componentName) String() string { - if i >= componentName(len(_componentName_index)-1) { - return "componentName(" + strconv.FormatInt(int64(i), 10) + ")" +func (i componentType) String() string { + if i >= componentType(len(_componentType_index)-1) { + return "componentType(" + strconv.FormatInt(int64(i), 10) + ")" } - return _componentName_name[_componentName_index[i]:_componentName_index[i+1]] + return _componentType_name[_componentType_index[i]:_componentType_index[i+1]] } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 3c168c2dbd6..51731e5daee 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -62,20 +62,20 @@ type ilmIndexSelector struct { st *indexState } -type componentName uint8 +type componentType uint8 -//go:generate stringer -linecomment -type componentName +//go:generate stringer -linecomment -type componentType const ( - componentTemplate componentName = iota //template + componentTemplate componentType = iota //template componentILM //ilm ) -type component struct { - name componentName +type feature struct { + component componentType enabled, overwrite, load bool } -func newComponent(name componentName, enabled, overwrite bool, mode LoadMode) component { +func newFeature(c componentType, enabled, overwrite bool, mode LoadMode) feature { if mode == LoadModeUnset && !enabled { mode = LoadModeDisabled } @@ -86,7 +86,7 @@ func newComponent(name componentName, enabled, overwrite bool, mode LoadMode) co enabled = true } load := mode.Enabled() && enabled - return component{name: name, enabled: enabled, overwrite: overwrite, load: load} + return feature{component: c, enabled: enabled, overwrite: overwrite, load: load} } func newIndexSupport( @@ -101,7 +101,7 @@ func newIndexSupport( ilmFactory = ilm.DefaultSupport } - ilm, err := ilmFactory(log, info, ilmConfig) + ilmSupporter, err := ilmFactory(log, info, ilmConfig) if err != nil { return nil, err } @@ -113,7 +113,7 @@ func newIndexSupport( return &indexSupport{ log: log, - ilm: ilm, + ilm: ilmSupporter, info: info, templateCfg: tmplCfg, migration: migration, @@ -125,7 +125,7 @@ func (s *indexSupport) Enabled() bool { return s.enabled(componentTemplate) || s.enabled(componentILM) } -func (s *indexSupport) enabled(c componentName) bool { +func (s *indexSupport) enabled(c componentType) bool { switch c { case componentTemplate: return s.templateCfg.Enabled @@ -139,10 +139,9 @@ func (s *indexSupport) Manager( clientHandler ClientHandler, assets Asseter, ) Manager { - ilm := s.ilm.Manager(clientHandler) return &indexManager{ support: s, - ilm: ilm, + ilm: s.ilm.Manager(clientHandler), clientHandler: clientHandler, assets: assets, } @@ -214,18 +213,22 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, }, nil } func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { - ilmComponent := newComponent(componentILM, m.support.enabled(componentILM), false, loadILM) - templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate), + ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), false, loadILM) + + templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) if ilmComponent.load && !templateComponent.load { return false, "Loading ILM policy and write alias without loading template " + "is not recommended. Check your configuration." } + if templateComponent.load && !ilmComponent.load && ilmComponent.enabled { - return false, "Loading template with ILM settings whithout loading ILM policy and alias can lead " + - "to issues and is not recommended. Check your configuration" + return false, "Loading template with ILM settings whithout loading ILM " + + "policy and alias can lead to issues and is not recommended. " + + "Check your configuration." } + var warn string if !ilmComponent.load { warn += "ILM policy and write alias loading not enabled. " @@ -247,8 +250,8 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Auto ILM enable success.") } - ilmComponent := newComponent(componentILM, withILM, false, loadILM) - templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate), + ilmComponent := newFeature(componentILM, withILM, false, loadILM) + templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) if ilmComponent.load { @@ -280,8 +283,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled if ilmComponent.enabled { - ilm := m.support.ilm - tmplCfg, err = applyILMSettings(log, tmplCfg, ilm.Policy(), ilm.Alias()) + tmplCfg, err = applyILMSettings(log, tmplCfg, m.support.ilm.Policy(), m.support.ilm.Alias()) if err != nil { return err } From 3b2cbaa91d0adec85bec61c72349c0f53325530d Mon Sep 17 00:00:00 2001 From: simitt Date: Thu, 9 May 2019 20:48:01 +0200 Subject: [PATCH 4/7] add new line --- libbeat/idxmgmt/std.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 51731e5daee..1fae6c6d8e3 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -212,6 +212,7 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, st: &s.st, }, nil } + func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), false, loadILM) From 9810cc55937f3706afbbce8f98ff6547229f81f9 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 9 May 2019 23:38:20 +0200 Subject: [PATCH 5/7] Add unit test Update mockClientHandler to implement the complete interface and record the order of 'create/load' operations. This is used to validate the index management unit tests for always checking the correct order of index management operations in the index manager. --- libbeat/idxmgmt/std_test.go | 94 +++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index b5b9c44eafb..07834d77b7b 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -32,6 +32,24 @@ import ( "github.com/elastic/beats/libbeat/template" ) +type mockClientHandler struct { + alias, policy string + expectsPolicy bool + + tmplCfg *template.TemplateConfig + tmplForce bool + + operations []mockCreateOp +} + +type mockCreateOp uint8 + +const ( + mockCreatePolicy mockCreateOp = iota + mockCreateTemplate + mockCreateAlias +) + func TestDefaultSupport_Enabled(t *testing.T) { cases := map[string]struct { ilmCalls []onCall @@ -397,69 +415,77 @@ func TestIndexManager_Setup(t *testing.T) { clientHandler := newMockClientHandler() manager := im.Manager(clientHandler, BeatsAssets([]byte("testbeat fields"))) err = manager.Setup(test.loadTemplate, test.loadILM) + clientHandler.assertInvariants(t) if test.err { assert.Error(t, err) } else { require.NoError(t, err) if test.tmplCfg == nil { - require.Nil(t, clientHandler.tl.tmplCfg) + assert.Nil(t, clientHandler.tmplCfg) + } else { - require.Equal(t, test.tmplCfg, clientHandler.tl.tmplCfg) + assert.Equal(t, test.tmplCfg, clientHandler.tmplCfg) } - assert.Equal(t, test.alias, clientHandler.il.alias) - assert.Equal(t, test.policy, clientHandler.il.policy) + assert.Equal(t, test.alias, clientHandler.alias) + assert.Equal(t, test.policy, clientHandler.policy) } }) } } -func newMockClientHandler() *mockClientHandler { - tl := mockTemplateLoader{} - il := mockILMClientHandler{} - return &mockClientHandler{&il, &tl, &tl, &il} +func (op mockCreateOp) String() string { + names := []string{"create-policy", "create-template", "create-alias"} + if int(op) > len(names) { + return "unknown" + } + return names[op] } -type mockClientHandler struct { - ilm.ClientHandler - template.Loader - - tl *mockTemplateLoader - il *mockILMClientHandler +func newMockClientHandler() *mockClientHandler { + return &mockClientHandler{} } -type mockTemplateLoader struct { - tmplCfg *template.TemplateConfig - force bool +func (h *mockClientHandler) Load(config template.TemplateConfig, _ beat.Info, fields []byte, migration bool) error { + h.recordOp(mockCreateTemplate) + h.tmplForce = config.Overwrite + h.tmplCfg = &config + return nil } -func (l *mockTemplateLoader) Load(config template.TemplateConfig, _ beat.Info, fields []byte, migration bool) error { - l.force = config.Overwrite - l.tmplCfg = &config - return nil +func (h *mockClientHandler) CheckILMEnabled(m ilm.Mode) (bool, error) { + return m == ilm.ModeEnabled || m == ilm.ModeAuto, nil } -type mockILMClientHandler struct { - alias, policy string +func (h *mockClientHandler) HasAlias(name string) (bool, error) { + return h.alias == name, nil } -func (ch *mockILMClientHandler) CheckILMEnabled(m ilm.Mode) (bool, error) { - return m == ilm.ModeEnabled || m == ilm.ModeAuto, nil +func (h *mockClientHandler) CreateAlias(alias ilm.Alias) error { + h.recordOp(mockCreateAlias) + h.alias = alias.Name + return nil } -func (ch *mockILMClientHandler) HasAlias(name string) (bool, error) { - return ch.alias == name, nil +func (h *mockClientHandler) HasILMPolicy(name string) (bool, error) { + return h.policy == name, nil } -func (ch *mockILMClientHandler) CreateAlias(alias ilm.Alias) error { - ch.alias = alias.Name +func (h *mockClientHandler) CreateILMPolicy(policy ilm.Policy) error { + h.recordOp(mockCreatePolicy) + h.policy = policy.Name return nil } -func (ch *mockILMClientHandler) HasILMPolicy(name string) (bool, error) { - return ch.policy == name, nil +func (h *mockClientHandler) recordOp(op mockCreateOp) { + h.operations = append(h.operations, op) } -func (ch *mockILMClientHandler) CreateILMPolicy(policy ilm.Policy) error { - ch.policy = policy.Name - return nil +func (h *mockClientHandler) assertInvariants(t *testing.T) { + for i, op := range h.operations { + for _, older := range h.operations[:i] { + if older > op { + t.Errorf("Operation: '%v' has been executed before '%v'", older, op) + } + } + } } From b5948f266acd3d906f8a6972585006080f4697ef Mon Sep 17 00:00:00 2001 From: simitt Date: Fri, 10 May 2019 11:30:59 +0200 Subject: [PATCH 6/7] Fix order of template and write alias creation. According to https://github.com/elastic/beats/pull/12149 --- libbeat/idxmgmt/idxmgmt.go | 2 ++ libbeat/idxmgmt/std.go | 23 +++++++------ libbeat/tests/system/idxmgmt.py | 17 ++++++---- .../system/test_cmd_setup_index_management.py | 32 ++++++++++++++----- libbeat/tests/system/test_ilm.py | 16 +++------- libbeat/tests/system/test_template.py | 10 +++--- 6 files changed, 57 insertions(+), 43 deletions(-) diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go index a18eae5d8ff..6837af03d13 100644 --- a/libbeat/idxmgmt/idxmgmt.go +++ b/libbeat/idxmgmt/idxmgmt.go @@ -62,6 +62,8 @@ type Asseter interface { // Elastic Stack. type Manager interface { VerifySetup(template, ilm LoadMode) (bool, string) + // When supporting index lifecycle management, ensure templates and policies + // are created before write aliases, to ensure templates are applied to the indices. Setup(template, ilm LoadMode) error } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 1fae6c6d8e3..13d1a201fcf 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -240,6 +240,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string return warn == "", warn } +// func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log := m.support.log @@ -267,16 +268,6 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { if policyCreated && templateComponent.enabled { templateComponent.overwrite = true } - - // create alias - if err := m.ilm.EnsureAlias(); err != nil { - if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { - return err - } - log.Info("Write alias exists already") - } else { - log.Info("Write alias successfully generated.") - } } if templateComponent.load { @@ -298,6 +289,18 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Loaded index template.") } + if ilmComponent.load { + // ensure alias is created after the template is created + if err := m.ilm.EnsureAlias(); err != nil { + if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { + return err + } + log.Info("Write alias exists already") + } else { + log.Info("Write alias successfully generated.") + } + } + return nil } diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 0c673ccfb28..836a6d11797 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -7,17 +7,20 @@ class IdxMgmt(object): def __init__(self, client, index): self._client = client - if index == "": - index == "mockbeat" - self._index = index + self._index = index if index != '' and index != '*' else 'mockbeat' def needs_init(self, s): return s == '' or s == '*' - def delete(self, index=""): - self.delete_index_and_alias(index=index) - self.delete_template(template=index) - self.delete_policy(policy=index) + def delete(self, indices=[]): + indices = list(filter(lambda x: x != '', indices)) + if not indices: + indices == [self._index] + for i in indices: + self.delete_index_and_alias(i) + self.delete_template(template=i) + for i in indices: + self.delete_policy(policy=i) def delete_index_and_alias(self, index=""): if self.needs_init(index): diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 8f556942adc..bcf6d7297b9 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -3,9 +3,9 @@ import os from nose.plugins.attrib import attr import unittest -import shutil -import datetime import logging +from nose.tools import raises +from elasticsearch import RequestError INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -28,17 +28,13 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -64,6 +60,26 @@ def test_setup_default(self): self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + @raises(RequestError) + def test_setup_default(self): + """ + Test setup --index-management with default config + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_policy_created(self.policy_name) + # try deleting policy needs to raise an error as it is in use + self.idxmgmt.delete_policy(self.policy_name) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') def test_setup_template_disabled(self): diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index aede356fd79..c73934ca972 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -21,14 +21,10 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -169,17 +165,13 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) def render_config(self, **kwargs): self.render_config_template( diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 03bf017ee06..084993b8850 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -128,10 +128,10 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def tearDown(self): - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -186,14 +186,12 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) def render_config(self, **kwargs): self.render_config_template( From 155969d8289730740878e85e25fca4e751825c44 Mon Sep 17 00:00:00 2001 From: simitt Date: Fri, 10 May 2019 14:52:34 +0200 Subject: [PATCH 7/7] minor changes --- libbeat/idxmgmt/std_test.go | 4 +++- libbeat/tests/system/test_cmd_setup_index_management.py | 4 ++-- libbeat/tests/system/test_ilm.py | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index 07834d77b7b..9a0b58c266c 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -259,10 +259,12 @@ func TestIndexManager_VerifySetup(t *testing.T) { require.NoError(t, err) support, err := MakeDefaultSupport(ilm.StdSupport)(nil, beat.Info{}, cfg) require.NoError(t, err) - manager := support.Manager(newMockClientHandler(), nil) + clientHandler := newMockClientHandler() + manager := support.Manager(clientHandler, nil) ok, warn := manager.VerifySetup(setup.loadTmpl, setup.loadILM) assert.Equal(t, setup.ok, ok) assert.Contains(t, warn, setup.warn) + clientHandler.assertInvariants(t) }) } } diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index bcf6d7297b9..8eab7ff3e0b 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -28,13 +28,13 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index c73934ca972..9195954600a 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -21,10 +21,10 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -165,13 +165,13 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.custom_policy, self.index_name]) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template(