Deprecate Monitoring Settings (#79499)

This PR deprecates all monitoring settings as well as adds deprecation info entries for each setting. Collecting and shipping monitoring data using the Monitoring plugin will be deprecated in 7.16 and will be removed at some point in the 8.x line after sufficient wait time. The recommended approach for collecting and shipping monitoring data going forward is to use Metricbeat. The recommended approach for alerting is Kibana alerting.
elastic · Oct 20, 2021 · 12d10b0 · 12d10b0
1 parent f7454b8
commit 12d10b0
Show file tree

Hide file tree

Showing 38 changed files with 662 additions and 175 deletions.
diff --git a/docs/reference/monitoring/collecting-monitoring-data.asciidoc b/docs/reference/monitoring/collecting-monitoring-data.asciidoc
@@ -55,14 +55,18 @@ For example, use the following APIs to review and change this setting:
 [source,console]
 ----------------------------------
 GET _cluster/settings
+----------------------------------
 
+[source,console]
+----------------------------------
 PUT _cluster/settings
 {
  "persistent": {
  "xpack.monitoring.collection.enabled": true
  }
 }
 ----------------------------------
+// TEST[warning:[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 Alternatively, you can enable this setting in {kib}. In the side navigation,
 click *Monitoring*. If data collection is disabled, you are prompted to turn it

diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc
@@ -26,14 +26,18 @@ You can use the following APIs to review and change this setting:
 [source,console]
 ----------------------------------
 GET _cluster/settings
+----------------------------------
 
+[source,console]
+----------------------------------
 PUT _cluster/settings
 {
  "persistent": {
  "xpack.monitoring.collection.enabled": true
  }
 }
 ----------------------------------
+// TEST[warning:[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 If {es} {security-features} are enabled, you must have `monitor` cluster privileges to 
 view the cluster settings and `manage` cluster privileges to change them.
@@ -201,6 +205,7 @@ PUT _cluster/settings
  }
 }
 ----------------------------------
+// TEST[warning:[xpack.monitoring.elasticsearch.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 
 If {es} {security-features} are enabled, you must have `monitor` cluster
 privileges to view the cluster settings and `manage` cluster privileges

diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc
@@ -277,6 +277,7 @@ PUT _cluster/settings
  }
 }
 ----
+// TEST[warning:[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.]
 // TEST[continued]
 ////
 

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -927,7 +927,8 @@ private void wipeClusterSettings() throws IOException {
  } else if (warnings.size() > 1) {
  return true;
  } else {
- return warnings.get(0).startsWith("[transient settings removal]") == false;
+ return warnings.get(0).startsWith("[transient settings removal]") == false &&
+ warnings.get(0).contains("xpack.monitoring") == false;
  }
  }));
 

diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle
@@ -34,9 +34,6 @@ testClusters.matching { it.name == "integTest" }.configureEach {
  setting 'xpack.security.enabled', 'true'
  setting 'xpack.security.authc.api_key.enabled', 'true'
  setting 'xpack.security.authc.token.enabled', 'true'
- // Disable monitoring exporters for the docs tests
- setting 'xpack.monitoring.exporters._local.type', 'local'
- setting 'xpack.monitoring.exporters._local.enabled', 'false'
  // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
  setting 'indices.lifecycle.history_index_enabled', 'false'
  setting 'xpack.license.self_generated.type', 'trial'

diff --git a/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java b/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java
@@ -120,11 +120,6 @@ protected boolean isWatcherTest() {
  return testName != null && (testName.contains("watcher/") || testName.contains("watcher\\"));
  }
 
- @Override
- protected boolean isMonitoringTest() {
- return false;
- }
-
  @Override
  protected boolean isMachineLearningTest() {
  String testName = getTestName();

diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
@@ -192,8 +192,6 @@ testClusters.configureEach {
  setting 'xpack.security.enabled', 'true'
  setting 'xpack.watcher.enabled', 'false'
  // Integration tests are supposed to enable/disable exporters before/after each test
- setting 'xpack.monitoring.exporters._local.type', 'local'
- setting 'xpack.monitoring.exporters._local.enabled', 'false'
  setting 'xpack.security.authc.token.enabled', 'true'
  setting 'xpack.security.authc.api_key.enabled', 'true'
  setting 'xpack.security.transport.ssl.enabled', 'true'

diff --git a/...r/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java b/...r/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java
@@ -149,6 +149,9 @@ public void testDoCollect() throws Exception {
  assertThat(document.getType(), is(AutoFollowStatsMonitoringDoc.TYPE));
  assertThat(document.getId(), nullValue());
  assertThat(document.stats(), is(autoFollowStats));
+
+ assertWarnings("[xpack.monitoring.collection.ccr.stats.timeout] setting was deprecated in Elasticsearch and will be removed in " +
+ "a future release! See the breaking changes documentation for the next major version.");
  }
 
  private List<FollowStatsAction.StatsResponse> mockStatuses() {

diff --git a/...ck/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringField.java b/...ck/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringField.java
@@ -31,7 +31,7 @@ public final class MonitoringField {
  public static final Setting<TimeValue> HISTORY_DURATION = timeSetting("xpack.monitoring.history.duration",
  TimeValue.timeValueHours(7 * 24), // default value (7 days)
  HISTORY_DURATION_MINIMUM, // minimum value
- Setting.Property.Dynamic, Setting.Property.NodeScope);
+ Setting.Property.Dynamic, Setting.Property.NodeScope, Setting.Property.Deprecated);
 
  private MonitoringField() {}
 }
diff --git a/...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
@@ -48,7 +48,38 @@ private DeprecationChecks() {
  NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting,
  NodeDeprecationChecks::checkExporterUseIngestPipelineSettings,
  NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting,
- NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting
+ NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting,
+ NodeDeprecationChecks::checkMonitoringSettingHistoryDuration,
+ NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery,
+ NodeDeprecationChecks::checkMonitoringSettingCollectIndices,
+ NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersHost,
+ NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername,
+ NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass,
+ NodeDeprecationChecks::checkMonitoringSettingExportersSSL,
+ NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase,
+ NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled,
+ NodeDeprecationChecks::checkMonitoringSettingExportersHeaders,
+ NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout,
+ NodeDeprecationChecks::checkMonitoringSettingExportersEnabled,
+ NodeDeprecationChecks::checkMonitoringSettingExportersType,
+ NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled,
+ NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist,
+ NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat,
+ NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts,
+ NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled,
+ NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled,
+ NodeDeprecationChecks::checkMonitoringSettingCollectionInterval
  );
 
  static List<Function<IndexMetadata, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(