Do not load ML jobs to 8.x from 7.x releases (#27771)

## What does this PR do?

This PR adds a check before loading ML assets to make sure, Beats are loading those to ES 7.x. If someone tries to load ML into ES 8.x, they are told to use the Machine Learning UI in Kibana.

This PR is opened against 7.x.

## Why is it important?

This work is part of the forward compatibility effort. We have to make sure users rely on the better solution for loading ML in Kibana, instead of the deprecated `setup --machine-learning` subcommand.

CHANGELOG.next.asciidoc

@@ -309,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update cloud.google.com/go library. {pull}28229[28229]
 - Update ECS to 1.12.0. {pull}27770[27770]
 - Fields mapped as `match_only_text` will automatically fallback to a `text` mapping when using Elasticsearch versions that do not support `match_only_text`. {pull}27770[27770]
+- Do not load ML jobs to Elasticsearch 8.x from new Beats 7.x releases. {pull}27771[27771]
 
 *Auditbeat*
 

filebeat/beater/filebeat.go

@@ -44,6 +44,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/kibana"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/management"
+	mlimporter "github.com/elastic/beats/v7/libbeat/ml-importer"
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 	"github.com/elastic/beats/v7/libbeat/outputs/elasticsearch"
 	"github.com/elastic/beats/v7/libbeat/publisher/pipetool"
@@ -160,8 +161,8 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *common.Config) (b
 	}
 
 	// register `setup` callback for ML jobs
-	b.SetupMLCallback = func(b *beat.Beat, kibanaConfig *common.Config) error {
-		return fb.loadModulesML(b, kibanaConfig)
+	b.SetupMLCallback = func(b *beat.Beat, fromFlag bool, kibanaConfig *common.Config) error {
+		return fb.loadModulesML(b, fromFlag, kibanaConfig)
 	}
 
 	err = fb.setupPipelineLoaderCallback(b)
@@ -223,7 +224,7 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error {
 	return err
 }
 
-func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) error {
+func (fb *Filebeat) loadModulesML(b *beat.Beat, fromFlag bool, kibanaConfig *common.Config) error {
 	var errs multierror.Errors
 
 	logp.Debug("machine-learning", "Setting up ML jobs for modules")
@@ -261,7 +262,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err
 		return errors.Errorf("Error creating Kibana client: %v", err)
 	}
 
-	if err := setupMLBasedOnVersion(fb.moduleRegistry, esClient, kibanaClient); err != nil {
+	if err := setupMLBasedOnVersion(fb.moduleRegistry, fromFlag, esClient, kibanaClient); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -287,17 +288,23 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err
 				continue
 			}
 
-			if err := setupMLBasedOnVersion(set, esClient, kibanaClient); err != nil {
+			if err := setupMLBasedOnVersion(set, fromFlag, esClient, kibanaClient); err != nil {
 				errs = append(errs, err)
 			}
 
 		}
 	}
+	if len(errs) == 0 {
+		fmt.Println("Loaded machine learning job configurations")
+	}
 
 	return errs.Err()
 }
 
-func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, esClient *eslegclient.Connection, kibanaClient *kibana.Client) error {
+func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, fromFlag bool, esClient *eslegclient.Connection, kibanaClient *kibana.Client) error {
+	if !mlimporter.IsCompatible(esClient) && fromFlag {
+		return fmt.Errorf("Machine learning jobs are not loaded because Elasticsearch version is too new. It must be 7.x for setting up ML using Beats. Use the Machine learning UI in Kibana.")
+	}
 	if isElasticsearchLoads(kibanaClient.GetVersion()) {
 		return reg.LoadML(esClient)
 	}

filebeat/fileset/modules.go

@@ -392,6 +392,11 @@ func checkAvailableProcessors(esClient PipelineLoader, requiredProcessors []Proc
 
 // LoadML loads the machine-learning configurations into Elasticsearch, if X-Pack is available
 func (reg *ModuleRegistry) LoadML(esClient PipelineLoader) error {
+	if !mlimporter.IsCompatible(esClient) {
+		logp.Info("Skipping loading machine learning jobs because of Elasticsearch version is too new.\nIt must be 7.x for setting up ML using Beats. Please use the Machine Learning UI in Kibana.")
+		return nil
+	}
+
 	haveXpack, err := mlimporter.HaveXpackML(esClient)
 	if err != nil {
 		return errors.Errorf("error checking if xpack is available: %v", err)
@@ -417,6 +422,11 @@ func (reg *ModuleRegistry) LoadML(esClient PipelineLoader) error {
 
 // SetupML sets up the machine-learning configurations into Elasticsearch using Kibana, if X-Pack is available
 func (reg *ModuleRegistry) SetupML(esClient PipelineLoader, kibanaClient *kibana.Client) error {
+	if !mlimporter.IsCompatible(esClient) {
+		logp.Info("Skipping loading machine learning jobs because of Elasticsearch version is too new.\nIt must be 7.x for setting up it using Beats. Please use the Machine Learning UI in Kibana.")
+		return nil
+	}
+
 	haveXpack, err := mlimporter.HaveXpackML(esClient)
 	if err != nil {
 		return errors.Errorf("Error checking if xpack is available: %v", err)

libbeat/beat/beat.go

@@ -90,7 +90,7 @@ type BeatConfig struct {
 
 // SetupMLCallback can be used by the Beat to register MachineLearning configurations
 // for the enabled modules.
-type SetupMLCallback func(*Beat, *common.Config) error
+type SetupMLCallback func(*Beat, bool, *common.Config) error
 
 // OverwritePipelinesCallback can be used by the Beat to register Ingest pipeline loader
 // for the enabled modules.

libbeat/cmd/instance/beat.go

@@ -508,6 +508,7 @@ type SetupSettings struct {
 	Template bool
 	//Deprecated: use IndexManagementKey instead
 	ILMPolicy bool
+	SetupAll  bool
 }
 
 // Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
@@ -573,11 +574,12 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
 		if setup.MachineLearning && b.SetupMLCallback != nil {
 			cfgwarn.Deprecate("8.0.0", "Setting up ML using %v is going to be removed. Please use the ML app to setup jobs.", strings.Title(b.Info.Beat))
 			fmt.Println("Setting up ML using setup --machine-learning is going to be removed in 8.0.0. Please use the ML app instead.\nSee more: https://www.elastic.co/guide/en/machine-learning/current/index.html")
-			err = b.SetupMLCallback(&b.Beat, b.Config.Kibana)
+			fmt.Println("It is not possble to load ML jobs into an Elasticsearch 8.0.0 or newer using the Beat.")
+
+			err = b.SetupMLCallback(&b.Beat, !setup.SetupAll, b.Config.Kibana)
 			if err != nil {
 				return err
 			}
-			fmt.Println("Loaded machine learning job configurations")
 		}
 
 		if setup.Pipeline && b.OverwritePipelinesCallback != nil {

libbeat/cmd/setup.go

@@ -94,7 +94,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
 			}
 
 			//create the struct to pass on
-			var s = instance.SetupSettings{}
+			var s = instance.SetupSettings{SetupAll: setupAll}
 			for k, v := range registeredFlags {
 				if setupAll || v {
 					switch k {

libbeat/docs/command-reference.asciidoc

@@ -752,7 +752,9 @@ endif::no_dashboards[]
 
 ifdef::has_ml_jobs[]
 * The machine learning jobs contain the configuration information and metadata
-necessary to analyze data for anomalies.
+necessary to analyze data for anomalies. You can use this flag to load machine learning jobs
+into Elasticsearch version 7. For version 8 and later, use the Machine learning UI in Kibana
+as  described in the {ml-docs}/create-jobs.html[documentation].
 endif::[]
 
 This command sets up the environment without actually running
@@ -783,6 +785,7 @@ Shows help for the `setup` command.
 
 ifdef::has_ml_jobs[]
 *`--machine-learning`*::
+deprecated:[7.12]
 Sets up machine learning job configurations only.
 endif::[]
 

libbeat/ml-importer/importer.go

@@ -40,6 +40,9 @@ var (
 	kibanaGetModuleURL    = "/api/ml/modules/get_module/%s"
 	kibanaRecognizeURL    = "/api/ml/modules/recognize/%s"
 	kibanaSetupModuleURL  = "/api/ml/modules/setup/%s"
+
+	// ML assets are not loaded to ES 8.0.0 or newer.
+	incompatibleSince = common.MustNewVersion("8.0.0")
 )
 
 // MLConfig contains the required configuration for loading one job and the associated
@@ -177,6 +180,13 @@ func ImportMachineLearningJob(esClient MLLoader, cfg *MLConfig) error {
 	return nil
 }
 
+// IsCompatible checks if the version of Elasticsearch is supported.
+// Beats does not load ML assets to 8.0 or newer.
+func IsCompatible(esClient MLLoader) bool {
+	esVersion := esClient.GetVersion()
+	return esVersion.LessThan(incompatibleSince)
+}
+
 // HaveXpackML checks whether X-pack is installed and has Machine Learning enabled.
 func HaveXpackML(esClient MLLoader) (bool, error) {
 	status, response, err := esClient.Request("GET", "/_xpack", "", nil, nil)