Removing xpack.monitoring.* settings (#18608)

* Removing xpack.monitoring.* setting and corresponding code

* Updating docs

* Adding CHANGELOG entry

* Removing ES calls to custom monitoring endpoints!

* Removing xpack monitoring API params

* Removing format from test

CHANGELOG.next.asciidoc

@@ -18,6 +18,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove the non-ECS `agent.hostname` field. Use the `agent.name` or `agent.id` fields for an identifier. {issue}16377[16377] {pull}18328[18328]
 - Make error message about locked data path actionable. {pull}18667[18667]
 - Ensure dynamic template names are unique for the same field. {pull}18849[18849]
+- Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608]
 
 *Auditbeat*
 

libbeat/cmd/instance/beat.go

@@ -888,12 +888,9 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err
 }
 
 func (b *Beat) setupMonitoring(settings Settings) (report.Reporter, error) {
-	monitoringCfg, reporterSettings, err := monitoring.SelectConfig(b.Config.MonitoringBeatConfig)
-	if err != nil {
-		return nil, err
-	}
+	monitoringCfg := b.Config.MonitoringBeatConfig.Monitoring
 
-	monitoringClusterUUID, err := monitoring.GetClusterUUID(b.Config.MonitoringBeatConfig.Monitoring)
+	monitoringClusterUUID, err := monitoring.GetClusterUUID(monitoringCfg)
 	if err != nil {
 		return nil, err
 	}
@@ -914,7 +911,6 @@ func (b *Beat) setupMonitoring(settings Settings) (report.Reporter, error) {
 
 		settings := report.Settings{
 			DefaultUsername: settings.Monitoring.DefaultUsername,
-			Format:          reporterSettings.Format,
 			ClusterUUID:     monitoringClusterUUID,
 		}
 		reporter, err := report.New(b.Info, settings, monitoringCfg, b.Config.Output)

libbeat/docs/monitoring/monitoring-beats.asciidoc

@@ -10,10 +10,10 @@
 
 You can use the {stack} {monitor-features} to gain insight into the health of
 ifndef::apm-server[]
-{beatname_uc} instances running in your environment. 
+{beatname_uc} instances running in your environment.
 endif::[]
 ifdef::apm-server[]
-{beatname_uc}. 
+{beatname_uc}.
 endif::[]
 
 To monitor {beatname_uc}, make sure monitoring is enabled on your {es} cluster,
@@ -23,18 +23,15 @@ of following methods:
 * <<monitoring-internal-collection,Internal collection>> - Internal
 collectors send monitoring data directly to your monitoring cluster.
 ifndef::serverless[]
-* <<monitoring-metricbeat-collection, {metricbeat} collection>> - 
+* <<monitoring-metricbeat-collection, {metricbeat} collection>> -
 {metricbeat} collects monitoring data from your {beatname_uc} instance
 and sends it directly to your monitoring cluster.
 endif::[]
-* <<monitoring-internal-collection-legacy,Legacy collection (deprecated)>> - 
-Legacy collectors send monitoring data to your production cluster.
-
 
 //Commenting out this link temporarily until the general monitoring docs can be
-//updated. 
-//To learn about monitoring in general, see 
-//{ref}/monitor-elasticsearch-cluster.html[Monitor a cluster]. 
+//updated.
+//To learn about monitoring in general, see
+//{ref}/monitor-elasticsearch-cluster.html[Monitor a cluster].
 
 --
 

libbeat/esleg/eslegclient/bulkapi.go

@@ -30,7 +30,6 @@ import (
 	"go.elastic.co/apm"
 	"go.elastic.co/apm/module/apmhttp"
 
-	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -94,43 +93,6 @@ func (conn *Connection) Bulk(
 	return conn.sendBulkRequest(requ)
 }
 
-// SendMonitoringBulk creates a HTTP request to the X-Pack Monitoring API containing a bunch of
-// operations and sends them to Elasticsearch. The request is retransmitted up to max_retries
-// before returning an error.
-func (conn *Connection) SendMonitoringBulk(
-	params map[string]string,
-	body []interface{},
-) (BulkResult, error) {
-	if len(body) == 0 {
-		return nil, nil
-	}
-
-	enc := conn.Encoder
-	enc.Reset()
-	if err := bulkEncode(conn.log, enc, body); err != nil {
-		return nil, err
-	}
-
-	if !conn.version.IsValid() {
-		if err := conn.Connect(); err != nil {
-			return nil, err
-		}
-	}
-
-	mergedParams := mergeParams(conn.ConnectionSettings.Parameters, params)
-
-	requ, err := newMonitoringBulkRequest(conn.GetVersion(), conn.URL, mergedParams, enc)
-	if err != nil {
-		return nil, err
-	}
-
-	_, result, err := conn.sendBulkRequest(requ)
-	if err != nil {
-		return nil, err
-	}
-	return result, nil
-}
-
 func newBulkRequest(
 	urlStr string,
 	index, docType string,
@@ -145,20 +107,6 @@ func newBulkRequest(
 	return newBulkRequestWithPath(urlStr, path, params, body)
 }
 
-func newMonitoringBulkRequest(
-	esVersion common.Version,
-	urlStr string,
-	params map[string]string,
-	body BodyEncoder,
-) (*bulkRequest, error) {
-	path, err := makePath("_monitoring", "bulk", "")
-	if err != nil {
-		return nil, err
-	}
-
-	return newBulkRequestWithPath(urlStr, path, params, body)
-}
-
 func newBulkRequestWithPath(
 	urlStr string,
 	path string,

libbeat/monitoring/monitoring.go

@@ -21,14 +21,11 @@ import (
 	"errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
-	"github.com/elastic/beats/v7/libbeat/monitoring/report"
 )
 
 // BeatConfig represents the part of the $BEAT.yml to do with monitoring settings
 type BeatConfig struct {
-	XPackMonitoring *common.Config `config:"xpack.monitoring"`
-	Monitoring      *common.Config `config:"monitoring"`
+	Monitoring *common.Config `config:"monitoring"`
 }
 
 type Mode uint8
@@ -42,11 +39,6 @@ const (
 	Full
 )
 
-var (
-	errMonitoringBothConfigEnabled = errors.New("both xpack.monitoring.* and monitoring.* cannot be set. Prefer to set monitoring.* and set monitoring.elasticsearch.hosts to monitoring cluster hosts")
-	warnMonitoringDeprecatedConfig = "xpack.monitoring.* settings are deprecated. Use monitoring.* instead, but set monitoring.elasticsearch.hosts to monitoring cluster hosts."
-)
-
 // Default is the global default metrics registry provided by the monitoring package.
 var Default = NewRegistry()
 
@@ -85,24 +77,6 @@ func Clear() error {
 	return Default.Clear()
 }
 
-// SelectConfig selects the appropriate monitoring configuration based on the user's settings in $BEAT.yml. Users may either
-// use xpack.monitoring.* settings OR monitoring.* settings but not both.
-func SelectConfig(beatCfg BeatConfig) (*common.Config, *report.Settings, error) {
-	switch {
-	case beatCfg.Monitoring.Enabled() && beatCfg.XPackMonitoring.Enabled():
-		return nil, nil, errMonitoringBothConfigEnabled
-	case beatCfg.XPackMonitoring.Enabled():
-		cfgwarn.Deprecate("8.0.0", warnMonitoringDeprecatedConfig)
-		monitoringCfg := beatCfg.XPackMonitoring
-		return monitoringCfg, &report.Settings{Format: report.FormatXPackMonitoringBulk}, nil
-	case beatCfg.Monitoring.Enabled():
-		monitoringCfg := beatCfg.Monitoring
-		return monitoringCfg, &report.Settings{Format: report.FormatBulk}, nil
-	default:
-		return nil, nil, nil
-	}
-}
-
 // GetClusterUUID returns the value of the monitoring.cluster_uuid setting, if it is set.
 func GetClusterUUID(monitoringCfg *common.Config) (string, error) {
 	if monitoringCfg == nil {