[cmd setup] Add and deprecate setup cmds for index handling

CHANGELOG-developer.next.asciidoc

@@ -37,3 +37,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Jinja2 version to 2.10.1. {pull}11817[11817]
 - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067]
 - Update urllib3 version to 1.24.2 {pull}11930[11930]
+- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]

libbeat/cmd/instance/beat.go

@@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error {
 
 //SetupSettings holds settings necessary for beat setup
 type SetupSettings struct {
- Template bool
  Dashboard bool
  MachineLearning bool
  Pipeline bool
- ILMPolicy bool
+ IndexManagement bool
+ //Deprecated: use IndexManagementKey instead
+ Template bool
+ //Deprecated: use IndexManagementKey instead
+ ILMPolicy bool
 }
 
 // Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
@@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
  return err
  }
 
- if setup.Template || setup.ILMPolicy {
+ if setup.IndexManagement || setup.Template || setup.ILMPolicy {
  outCfg := b.Config.Output
-
  if outCfg.Name() != "elasticsearch" {
  return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled")
  }
+ esClient, err := elasticsearch.NewConnectedClient(outCfg.Config())
+ if err != nil {
+ return err
+ }
 
- esConfig := outCfg.Config()
- if b.IdxSupporter.Enabled() {
- esClient, err := elasticsearch.NewConnectedClient(esConfig)
- if err != nil {
- return err
- }
-
- // prepare index by loading templates, lifecycle policies and write aliases
-
- m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
- var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
- if setup.Template {
- tmplLoadMode = idxmgmt.LoadModeOverwrite
- }
- if setup.ILMPolicy {
- ilmLoadMode = idxmgmt.LoadModeOverwrite
- }
-
- err = m.Setup(tmplLoadMode, ilmLoadMode)
- if err != nil {
- return err
- }
+ var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
+ if setup.IndexManagement || setup.Template {
+ loadTemplate = idxmgmt.LoadModeOverwrite
+ }
+ if setup.IndexManagement || setup.ILMPolicy {
+ loadILM = idxmgmt.LoadModeOverwrite
+ }
+ m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
+ if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {
+ fmt.Println(warn)
+ }
+ if err = m.Setup(loadTemplate, loadILM); err != nil {
+ return err
  }
- fmt.Println("Index setup complete.")
+ fmt.Println("Index setup finished.")
  }
 
  if setup.Dashboard {

libbeat/cmd/setup.go

@@ -28,15 +28,23 @@ import (
 )
 
 const (
- //TemplateKey used for defining template in setup cmd
- TemplateKey = "template"
  //DashboardKey used for registering dashboards in setup cmd
  DashboardKey = "dashboards"
  //MachineLearningKey used for registering ml jobs in setup cmd
  MachineLearningKey = "machine-learning"
  //PipelineKey used for registering pipelines in setup cmd
  PipelineKey = "pipelines"
- //ILMPolicyKey used for registering ilm in setup cmd
+ //IndexManagementKey used for loading all components related to ES index management in setup cmd
+ IndexManagementKey = "index-management"
+
+ //TemplateKey used for loading template in setup cmd
+ //
+ //Deprecated: use IndexManagementKey instead
+ TemplateKey = "template"
+
+ //ILMPolicyKey used for loading ilm in setup cmd
+ //
+ //Deprecated: use IndexManagementKey instead
  ILMPolicyKey = "ilm-policy"
 )
 
@@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  }
 
  var registeredFlags = map[string]bool{
- TemplateKey: false,
  DashboardKey: false,
  MachineLearningKey: false,
  PipelineKey: false,
+ IndexManagementKey: false,
+ TemplateKey: false,
  ILMPolicyKey: false,
  }
  var setupAll = true
@@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  for k, v := range registeredFlags {
  if setupAll || v {
  switch k {
- case TemplateKey:
- s.Template = true
  case DashboardKey:
  s.Dashboard = true
  case MachineLearningKey:
  s.MachineLearning = true
  case PipelineKey:
  s.Pipeline = true
+ case IndexManagementKey:
+ s.IndexManagement = true
  case ILMPolicyKey:
  s.ILMPolicy = true
+ case TemplateKey:
+ s.Template = true
  }
  }
  }
@@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  },
  }
 
- setup.Flags().Bool(TemplateKey, false, "Setup index template")
  setup.Flags().Bool(DashboardKey, false, "Setup dashboards")
  setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations")
  setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines")
+ setup.Flags().Bool(IndexManagementKey, false,
+ "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias")
+ setup.Flags().Bool(TemplateKey, false, "Setup index template")
+ setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
  setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy")
+ setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 
  return &setup
 }

libbeat/idxmgmt/idxmgmt.go

@@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
 // A manager instantiated via Supporter is responsible for instantiating/configuring
 // the index throughout the Elastic Stack.
 type Supporter interface {
- // Enalbed checks if index management is configured to configure templates,
- // ILM, or aliases.
+ // Enabled checks if index management is configured to setup templates or ILM
  Enabled() bool
 
  // BuildSelector create an index selector.
@@ -62,6 +61,7 @@ type Asseter interface {
 // Manager is used to initialize indices, ILM policies, and aliases within the
 // Elastic Stack.
 type Manager interface {
+ VerifySetup(template, ilm LoadMode) (bool, string)
  Setup(template, ilm LoadMode) error
 }
 

libbeat/idxmgmt/std.go

@@ -62,6 +62,33 @@ type ilmIndexSelector struct {
  st *indexState
 }
 
+type componentName uint8
+
+//go:generate stringer -linecomment -type componentName
+const (
+ componentTemplate componentName = iota //template
+ componentILM //ilm
+)
+
+type component struct {
+ name componentName
+ enabled, overwrite, load bool
+}
+
+func newComponent(name componentName, enabled, overwrite bool, mode LoadMode) component {
+ if mode == LoadModeUnset && !enabled {
+ mode = LoadModeDisabled
+ }
+ if mode >= LoadModeOverwrite {
+ overwrite = true
+ }
+ if mode == LoadModeForce {
+ enabled = true
+ }
+ load := mode.Enabled() && enabled
+ return component{name: name, enabled: enabled, overwrite: overwrite, load: load}
+}
+
 func newIndexSupport(
  log *logp.Logger,
  info beat.Info,
@@ -95,7 +122,17 @@ func newIndexSupport(
 }
 
 func (s *indexSupport) Enabled() bool {
- return s.templateCfg.Enabled || (s.ilm.Mode() != ilm.ModeDisabled)
+ return s.enabled(componentTemplate) || s.enabled(componentILM)
+}
+
+func (s *indexSupport) enabled(c componentName) bool {
+ switch c {
+ case componentTemplate:
+ return s.templateCfg.Enabled
+ case componentILM:
+ return s.ilm.Mode() != ilm.ModeDisabled
+ }
+ return false
 }
 
 func (s *indexSupport) Manager(
@@ -176,41 +213,55 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector,
  st: &s.st,
  }, nil
 }
+func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) {
+ ilmComponent := newComponent(componentILM, m.support.enabled(componentILM), false, loadILM)
+ templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate),
+ m.support.templateCfg.Overwrite, loadTemplate)
+
+ if ilmComponent.load && !templateComponent.load {
+ return false, "Loading ILM policy and write alias without loading template " +
+ "is not recommended. Check your configuration."
+ }
+ if templateComponent.load && !ilmComponent.load && ilmComponent.enabled {
+ return false, "Loading template with ILM settings whithout loading ILM policy and alias can lead " +
+ "to issues and is not recommended. Check your configuration"
+ }
+ var warn string
+ if !ilmComponent.load {
+ warn += "ILM policy and write alias loading not enabled. "
+ }
+ if !templateComponent.load {
+ warn += "Template loading not enabled."
+ }
+ return warn == "", warn
+}
 
 func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
- var err error
  log := m.support.log
 
- withILM := m.support.st.withILM.Load()
- if !withILM {
- withILM, err = m.ilm.Enabled()
- if err != nil {
- return err
- }
+ withILM, err := m.setupWithILM()
+ if err != nil {
+ return err
  }
- if loadILM == LoadModeUnset {
- if withILM {
- loadILM = LoadModeEnabled
- log.Info("Auto ILM enable success.")
- } else {
- loadILM = LoadModeDisabled
- }
+ if withILM && loadILM.Enabled() {
+ log.Info("Auto ILM enable success.")
  }
 
- if loadILM == LoadModeForce || withILM && loadILM.Enabled() {
-  // mark ILM as enabled in indexState if withILM is true
- m.support.st.withILM.CAS(false, true)
+ ilmComponent := newComponent(componentILM, withILM, false, loadILM)
+ templateComponent := newComponent(componentTemplate, m.support.enabled(componentTemplate),
+ m.support.templateCfg.Overwrite, loadTemplate)
 
+ if ilmComponent.load {
  // install ilm policy
- policyCreated, err := m.ilm.EnsurePolicy(loadILM >= LoadModeOverwrite)
+ policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite)
  if err != nil {
  return err
  }
  log.Info("ILM policy successfully loaded.")
 
  // The template should be updated if a new policy is created.
- if policyCreated && loadTemplate.Enabled() {
- loadTemplate = LoadModeOverwrite
+ if policyCreated && templateComponent.enabled {
+ templateComponent.overwrite = true
  }
 
  // create alias
@@ -224,24 +275,17 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
  }
  }
 
- // create and install template
- if loadTemplate == LoadModeForce || m.support.templateCfg.Enabled && loadTemplate.Enabled() {
+ if templateComponent.load {
  tmplCfg := m.support.templateCfg
+ tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled
 
- if withILM {
+ if ilmComponent.enabled {
  ilm := m.support.ilm
  tmplCfg, err = applyILMSettings(log, tmplCfg, ilm.Policy(), ilm.Alias())
  if err != nil {
  return err
  }
  }
-
- if loadTemplate == LoadModeForce {
- tmplCfg.Enabled = true
- }
- if loadTemplate >= LoadModeOverwrite {
- tmplCfg.Overwrite = true
- }
  fields := m.assets.Fields(m.support.info.Beat)
  err = m.clientHandler.Load(tmplCfg, m.support.info, fields, m.support.migration)
  if err != nil {
@@ -254,6 +298,22 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
  return nil
 }
 
+func (m *indexManager) setupWithILM() (bool, error) {
+ var err error
+ withILM := m.support.st.withILM.Load()
+ if !withILM {
+ withILM, err = m.ilm.Enabled()
+ if err != nil {
+ return false, err
+ }
+ if withILM {
+ // mark ILM as enabled in indexState
+ m.support.st.withILM.CAS(false, true)
+ }
+ }
+ return withILM, nil
+}
+
 func (s *ilmIndexSelector) Select(evt *beat.Event) (string, error) {
  if idx := getEventCustomIndex(evt); idx != "" {
  return idx, nil