[cmd setup] Add and deprecate setup cmds for index handling

CHANGELOG-developer.next.asciidoc

@@ -38,3 +38,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067]
 - Update urllib3 version to 1.24.2 {pull}11930[11930]
 - Add libbeat/common/cleanup package. {pull}12134[12134]
+- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]

libbeat/cmd/instance/beat.go

@@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error {
 
 //SetupSettings holds settings necessary for beat setup
 type SetupSettings struct {
- Template bool
  Dashboard bool
  MachineLearning bool
  Pipeline bool
- ILMPolicy bool
+ IndexManagement bool
+ //Deprecated: use IndexManagementKey instead
+ Template bool
+ //Deprecated: use IndexManagementKey instead
+ ILMPolicy bool
 }
 
 // Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
@@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
  return err
  }
 
- if setup.Template || setup.ILMPolicy {
+ if setup.IndexManagement || setup.Template || setup.ILMPolicy {
  outCfg := b.Config.Output
-
  if outCfg.Name() != "elasticsearch" {
  return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled")
  }
+ esClient, err := elasticsearch.NewConnectedClient(outCfg.Config())
+ if err != nil {
+ return err
+ }
 
- esConfig := outCfg.Config()
- if b.IdxSupporter.Enabled() {
- esClient, err := elasticsearch.NewConnectedClient(esConfig)
- if err != nil {
- return err
- }
-
- // prepare index by loading templates, lifecycle policies and write aliases
-
- m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
- var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
- if setup.Template {
- tmplLoadMode = idxmgmt.LoadModeOverwrite
- }
- if setup.ILMPolicy {
- ilmLoadMode = idxmgmt.LoadModeOverwrite
- }
-
- err = m.Setup(tmplLoadMode, ilmLoadMode)
- if err != nil {
- return err
- }
+ var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
+ if setup.IndexManagement || setup.Template {
+ loadTemplate = idxmgmt.LoadModeOverwrite
+ }
+ if setup.IndexManagement || setup.ILMPolicy {
+ loadILM = idxmgmt.LoadModeOverwrite
+ }
+ m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
+ if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {
+ fmt.Println(warn)
+ }
+ if err = m.Setup(loadTemplate, loadILM); err != nil {
+ return err
  }
- fmt.Println("Index setup complete.")
+ fmt.Println("Index setup finished.")
  }
 
  if setup.Dashboard {

libbeat/cmd/setup.go

@@ -28,15 +28,23 @@ import (
 )
 
 const (
- //TemplateKey used for defining template in setup cmd
- TemplateKey = "template"
  //DashboardKey used for registering dashboards in setup cmd
  DashboardKey = "dashboards"
  //MachineLearningKey used for registering ml jobs in setup cmd
  MachineLearningKey = "machine-learning"
  //PipelineKey used for registering pipelines in setup cmd
  PipelineKey = "pipelines"
- //ILMPolicyKey used for registering ilm in setup cmd
+ //IndexManagementKey used for loading all components related to ES index management in setup cmd
+ IndexManagementKey = "index-management"
+
+ //TemplateKey used for loading template in setup cmd
+ //
+ //Deprecated: use IndexManagementKey instead
+ TemplateKey = "template"
+
+ //ILMPolicyKey used for loading ilm in setup cmd
+ //
+ //Deprecated: use IndexManagementKey instead
  ILMPolicyKey = "ilm-policy"
 )
 
@@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  }
 
  var registeredFlags = map[string]bool{
- TemplateKey: false,
  DashboardKey: false,
  MachineLearningKey: false,
  PipelineKey: false,
+ IndexManagementKey: false,
+ TemplateKey: false,
  ILMPolicyKey: false,
  }
  var setupAll = true
@@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  for k, v := range registeredFlags {
  if setupAll || v {
  switch k {
- case TemplateKey:
- s.Template = true
  case DashboardKey:
  s.Dashboard = true
  case MachineLearningKey:
  s.MachineLearning = true
  case PipelineKey:
  s.Pipeline = true
+ case IndexManagementKey:
+ s.IndexManagement = true
  case ILMPolicyKey:
  s.ILMPolicy = true
+ case TemplateKey:
+ s.Template = true
  }
  }
  }
@@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  },
  }
 
- setup.Flags().Bool(TemplateKey, false, "Setup index template")
  setup.Flags().Bool(DashboardKey, false, "Setup dashboards")
  setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations")
  setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines")
+ setup.Flags().Bool(IndexManagementKey, false,
+ "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias")
+ setup.Flags().Bool(TemplateKey, false, "Setup index template")
+ setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
  setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy")
+ setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
 
  return &setup
 }

libbeat/idxmgmt/idxmgmt.go

@@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
 // A manager instantiated via Supporter is responsible for instantiating/configuring
 // the index throughout the Elastic Stack.
 type Supporter interface {
- // Enalbed checks if index management is configured to configure templates,
- // ILM, or aliases.
+ // Enabled checks if index management is configured to setup templates or ILM
  Enabled() bool
 
  // BuildSelector create an index selector.
@@ -62,6 +61,9 @@ type Asseter interface {
 // Manager is used to initialize indices, ILM policies, and aliases within the
 // Elastic Stack.
 type Manager interface {
+ VerifySetup(template, ilm LoadMode) (bool, string)
+ // When supporting index lifecycle management, ensure templates and policies
+ // are created before write aliases, to ensure templates are applied to the indices.
  Setup(template, ilm LoadMode) error
 }