diff --git a/docs/changelog/86524.yaml b/docs/changelog/86524.yaml new file mode 100644 index 0000000000000..35652fffcd74e --- /dev/null +++ b/docs/changelog/86524.yaml @@ -0,0 +1,5 @@ +pr: 86524 +summary: Master stability health indicator part 1 (when a master has been seen recently) +area: Health +type: feature +issues: [] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml index 685ce85976b33..5e4327d0d5177 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml @@ -1,8 +1,8 @@ --- "cluster health basic test": - skip: - version: "- 8.2.99" - reason: "summary text was updated in 8.3.0" + version: "- 8.3.99" + reason: "health was only added in 8.2.0, and master_is_stable in 8.4.0" - do: _internal.health: {} @@ -10,5 +10,5 @@ - is_true: cluster_name - match: { status: "green" } - match: { components.cluster_coordination.status: "green" } - - match: { components.cluster_coordination.indicators.instance_has_master.status: "green" } - - match: { components.cluster_coordination.indicators.instance_has_master.summary: "Health coordinating instance has an elected master node." } + - match: { components.cluster_coordination.indicators.master_is_stable.status: "green" } + - match: { components.cluster_coordination.indicators.master_is_stable.summary: "The cluster has a stable master node" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/20_component.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/20_component.yml index 174e0a6e1f4ab..18b4bdc9dfa2b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/20_component.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/20_component.yml @@ -1,8 +1,8 @@ --- "cluster health test drilling down into a component": - skip: - version: "- 8.2.99" - reason: "health drilldown was only added in 8.3.0" + version: "- 8.3.99" + reason: "health drilldown was only added in 8.3.0, and master_is_stable in 8.4.0" - do: _internal.health: @@ -10,9 +10,7 @@ - is_true: cluster_name - match: { components.cluster_coordination.status: "green" } - - match: { components.cluster_coordination.indicators.instance_has_master.status: "green" } - - match: { components.cluster_coordination.indicators.instance_has_master.summary: "Health coordinating instance has an elected master node." } - - is_true: components.cluster_coordination.indicators.instance_has_master.details.coordinating_node.node_id - - is_true: components.cluster_coordination.indicators.instance_has_master.details.coordinating_node.name - - is_true: components.cluster_coordination.indicators.instance_has_master.details.master_node.node_id - - is_true: components.cluster_coordination.indicators.instance_has_master.details.master_node.name + - match: { components.cluster_coordination.indicators.master_is_stable.status: "green" } + - match: { components.cluster_coordination.indicators.master_is_stable.summary: "The cluster has a stable master node" } + - is_true: components.cluster_coordination.indicators.master_is_stable.details.current_master + - is_true: components.cluster_coordination.indicators.master_is_stable.details.recent_masters diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml index 3a531a9f36bc2..aa077b0dd78a3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml @@ -1,18 +1,16 @@ --- "cluster health test drilling down into a component and a feature": - skip: - version: "- 8.2.99" - reason: "health drilldown was only added in 8.3.0" + version: "- 8.3.99" + reason: "health drilldown was only added in 8.3.0, and master_is_stable in 8.4.0" - do: _internal.health: component: cluster_coordination - feature: instance_has_master + feature: master_is_stable - is_true: cluster_name - - match: { components.cluster_coordination.indicators.instance_has_master.status: "green" } - - match: { components.cluster_coordination.indicators.instance_has_master.summary: "Health coordinating instance has an elected master node." } - - is_true: components.cluster_coordination.indicators.instance_has_master.details.coordinating_node.node_id - - is_true: components.cluster_coordination.indicators.instance_has_master.details.coordinating_node.name - - is_true: components.cluster_coordination.indicators.instance_has_master.details.master_node.node_id - - is_true: components.cluster_coordination.indicators.instance_has_master.details.master_node.name + - match: { components.cluster_coordination.indicators.master_is_stable.status: "green" } + - match: { components.cluster_coordination.indicators.master_is_stable.summary: "The cluster has a stable master node" } + - is_true: components.cluster_coordination.indicators.master_is_stable.details.current_master + - is_true: components.cluster_coordination.indicators.master_is_stable.details.recent_masters diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorServiceIT.java deleted file mode 100644 index 8b73db7e4cbbc..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorServiceIT.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.cluster.coordination; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.health.GetHealthAction; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.disruption.NetworkDisruption; -import org.elasticsearch.test.transport.MockTransportService; - -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import static org.elasticsearch.cluster.coordination.InstanceHasMasterHealthIndicatorService.NAME; -import static org.elasticsearch.health.HealthStatus.GREEN; -import static org.elasticsearch.health.HealthStatus.RED; -import static org.elasticsearch.health.ServerHealthComponents.CLUSTER_COORDINATION; -import static org.hamcrest.Matchers.equalTo; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) -public class InstanceHasMasterHealthIndicatorServiceIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return List.of(MockTransportService.TestPlugin.class); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(NoMasterBlockService.NO_MASTER_BLOCK_SETTING.getKey(), "all") - .build(); - } - - public void testGetHealthWhenMasterIsElected() throws Exception { - var client = client(); - - var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(randomBoolean())).get(); - - assertThat(response.findComponent(CLUSTER_COORDINATION).findIndicator(NAME).status(), equalTo(GREEN)); - } - - public void testGetHealthWhenNoMaster() throws Exception { - var client = internalCluster().coordOnlyNodeClient(); - - var disruptionScheme = new NetworkDisruption( - new NetworkDisruption.IsolateAllNodes(Set.of(internalCluster().getNodeNames())), - NetworkDisruption.DISCONNECT - ); - - internalCluster().setDisruptionScheme(disruptionScheme); - disruptionScheme.startDisrupting(); - - try { - assertBusy(() -> { - ClusterState state = client.admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID)); - - var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(randomBoolean())).get(); - - assertThat(response.findComponent(CLUSTER_COORDINATION).findIndicator(NAME).status(), equalTo(RED)); - }); - } finally { - internalCluster().clearDisruptionScheme(true); - } - } -} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java index 9d72e01fc59b4..73c1fda9274dd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java @@ -9,18 +9,23 @@ package org.elasticsearch.discovery; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.FollowersChecker; import org.elasticsearch.cluster.coordination.LeaderChecker; +import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Tuple; +import org.elasticsearch.health.GetHealthAction; +import org.elasticsearch.health.HealthStatus; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.LongGCDisruption; @@ -29,7 +34,12 @@ import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions; import org.elasticsearch.test.disruption.SingleNodeDisruption; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -41,9 +51,11 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; /** @@ -90,6 +102,8 @@ public void testFailWithMinimumMasterNodesConfigured() throws Exception { // continuously ping until network failures have been resolved. However // It may a take a bit before the node detects it has been cut off from the elected master ensureNoMaster(unluckyNode); + // because it has had a master within the last 30s: + assertGreenMasterStability(internalCluster().client(unluckyNode)); networkDisconnect.stopDisrupting(); @@ -98,6 +112,29 @@ public void testFailWithMinimumMasterNodesConfigured() throws Exception { // The elected master shouldn't have changed, since the unlucky node never could have elected itself as master assertThat(internalCluster().getMasterName(), equalTo(masterNode)); + assertGreenMasterStability(internalCluster().client()); + } + + private void assertGreenMasterStability(Client client) throws ExecutionException, InterruptedException, IOException { + assertMasterStability(client, HealthStatus.GREEN, "The cluster has a stable master node"); + } + + private void assertMasterStability(Client client, HealthStatus expectedStatus, String expectedSummarySubstring) + throws ExecutionException, InterruptedException, IOException { + GetHealthAction.Response healthResponse = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(true)).get(); + String debugInformation = xContentToString(healthResponse); + assertThat(debugInformation, healthResponse.getStatus(), equalTo(expectedStatus)); + assertThat( + debugInformation, + healthResponse.findComponent("cluster_coordination").findIndicator("master_is_stable").summary(), + containsString(expectedSummarySubstring) + ); + } + + private String xContentToString(ToXContentObject xContent) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder(); + xContent.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); } private void ensureNoMaster(String node) throws Exception { @@ -113,6 +150,7 @@ private void ensureNoMaster(String node) throws Exception { */ public void testFollowerCheckerDetectsDisconnectedNodeAfterMasterReelection() throws Exception { testFollowerCheckerAfterMasterReelection(NetworkDisruption.DISCONNECT, Settings.EMPTY); + assertGreenMasterStability(internalCluster().client()); } /** @@ -129,6 +167,7 @@ public void testFollowerCheckerDetectsUnresponsiveNodeAfterMasterReelection() th .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "10s") .build() ); + assertGreenMasterStability(internalCluster().client()); } private void testFollowerCheckerAfterMasterReelection(NetworkLinkDisruptionType networkLinkDisruptionType, Settings settings) @@ -264,6 +303,162 @@ public void onFailure(Exception e) { transitions.stream().noneMatch(t -> oldMasterNode.equals(t.v2())) ); } + assertGreenMasterStability(internalCluster().client()); } + /** + * This helper method creates a 3-node cluster where all nodes are master-eligible, and then simulates a long GC on the master node 5 + * times (forcing another node to be elected master 5 times). It then asserts that the master stability health indicator status is + * YELLOW, and that expectedMasterStabilitySummarySubstring is contained in the summary. + * @param expectedMasterStabilitySummarySubstring A string to expect in the master stability health indictor summary + * @throws Exception + */ + public void testRepeatedMasterChanges(String expectedMasterStabilitySummarySubstring) throws Exception { + final List nodes = internalCluster().startNodes( + 3, + Settings.builder() + .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") + .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") + .put(StableMasterHealthIndicatorService.IDENTITY_CHANGES_THRESHOLD_SETTING.getKey(), 1) + .put(StableMasterHealthIndicatorService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 100) + .build() + ); + ensureStableCluster(3); + String firstMaster = internalCluster().getMasterName(); + // Force the master to change 2 times: + for (int i = 0; i < 2; i++) { + // Save the current master node as old master node, because that node will get frozen + final String oldMasterNode = internalCluster().getMasterName(); + + // Simulating a painful gc by suspending all threads for a long time on the current elected master node. + SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), oldMasterNode); + + // Save the majority side + final List majoritySide = new ArrayList<>(nodes); + majoritySide.remove(oldMasterNode); + + // Keeps track of the previous and current master when a master node transition took place on each node on the majority side: + final Map>> masters = Collections.synchronizedMap(new HashMap<>()); + for (final String node : majoritySide) { + masters.put(node, new ArrayList<>()); + internalCluster().getInstance(ClusterService.class, node).addListener(event -> { + DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode(); + DiscoveryNode currentMaster = event.state().nodes().getMasterNode(); + if (Objects.equals(previousMaster, currentMaster) == false) { + logger.info( + "--> node {} received new cluster state: {} \n and had previous cluster state: {}", + node, + event.state(), + event.previousState() + ); + String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null; + String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null; + masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName)); + } + }); + } + + final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1); + internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> { + if (event.state().nodes().getMasterNodeId() == null) { + oldMasterNodeSteppedDown.countDown(); + } + }); + internalCluster().clearDisruptionScheme(); + internalCluster().setDisruptionScheme(masterNodeDisruption); + logger.info("--> freezing node [{}]", oldMasterNode); + masterNodeDisruption.startDisrupting(); + + // Wait for majority side to elect a new master + assertBusy(() -> { + for (final Map.Entry>> entry : masters.entrySet()) { + final List> transitions = entry.getValue(); + assertTrue(entry.getKey() + ": " + transitions, transitions.stream().anyMatch(transition -> transition.v2() != null)); + } + }); + + // Save the new elected master node + final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0)); + logger.info("--> new detected master node [{}]", newMasterNode); + + // Stop disruption + logger.info("--> unfreezing node [{}]", oldMasterNode); + masterNodeDisruption.stopDisrupting(); + + oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS); + logger.info("--> [{}] stepped down as master", oldMasterNode); + ensureStableCluster(3); + + assertThat(masters.size(), equalTo(2)); + } + List nodeNamesExceptFirstMaster = Arrays.stream(internalCluster().getNodeNames()) + .filter(name -> name.equals(firstMaster) == false) + .toList(); + /* + * It is possible that the first node that became master got re-elected repeatedly. And since it was in a simulated GC when the + * other node(s) were master, it only saw itself as master. So we want to check with another node. + */ + Client client = internalCluster().client(randomFrom(nodeNamesExceptFirstMaster)); + assertMasterStability(client, HealthStatus.YELLOW, expectedMasterStabilitySummarySubstring); + } + + public void testRepeatedNullMasterRecognizedAsGreenIfMasterDoesNotKnowItIsUnstable() throws Exception { + /* + * In this test we have a single master-eligible node. We pause it repeatedly (simulating a long GC pause for example) so that + * other nodes decide it is no longer the master. However since there is no other master-eligible node, another node is never + * elected master. And the master node never recognizes that it had a problem. So when we run the master stability check on one + * of the data nodes, it will see that there is a problem (the master has gone null repeatedly), but when it checks with the + * master, the master says everything is fine. So we expect a GREEN status. + */ + final List masterNodes = internalCluster().startMasterOnlyNodes( + 1, + Settings.builder() + .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") + .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") + .put(StableMasterHealthIndicatorService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 1) + .build() + ); + final List dataNodes = internalCluster().startDataOnlyNodes( + 2, + Settings.builder() + .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") + .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") + .put(StableMasterHealthIndicatorService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 1) + .build() + ); + ensureStableCluster(3); + for (int i = 0; i < 2; i++) { + final String masterNode = masterNodes.get(0); + + // Simulating a painful gc by suspending all threads for a long time on the current elected master node. + SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), masterNode); + + final CountDownLatch dataNodeMasterSteppedDown = new CountDownLatch(2); + internalCluster().getInstance(ClusterService.class, masterNode).addListener(event -> { + if (event.state().nodes().getMasterNodeId() == null) { + dataNodeMasterSteppedDown.countDown(); + } + }); + internalCluster().getInstance(ClusterService.class, dataNodes.get(0)).addListener(event -> { + if (event.state().nodes().getMasterNodeId() == null) { + dataNodeMasterSteppedDown.countDown(); + } + }); + internalCluster().getInstance(ClusterService.class, dataNodes.get(1)).addListener(event -> { + if (event.state().nodes().getMasterNodeId() == null) { + dataNodeMasterSteppedDown.countDown(); + } + }); + internalCluster().clearDisruptionScheme(); + internalCluster().setDisruptionScheme(masterNodeDisruption); + logger.info("--> freezing node [{}]", masterNode); + masterNodeDisruption.startDisrupting(); + dataNodeMasterSteppedDown.await(30, TimeUnit.SECONDS); + // Stop disruption + logger.info("--> unfreezing node [{}]", masterNode); + masterNodeDisruption.stopDisrupting(); + ensureStableCluster(3); + } + assertGreenMasterStability(internalCluster().client(randomFrom(dataNodes))); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorService.java deleted file mode 100644 index d6dc58e484d06..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/InstanceHasMasterHealthIndicatorService.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.cluster.coordination; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.health.HealthIndicatorDetails; -import org.elasticsearch.health.HealthIndicatorImpact; -import org.elasticsearch.health.HealthIndicatorResult; -import org.elasticsearch.health.HealthIndicatorService; -import org.elasticsearch.health.HealthStatus; -import org.elasticsearch.health.ImpactArea; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.elasticsearch.health.ServerHealthComponents.CLUSTER_COORDINATION; - -public class InstanceHasMasterHealthIndicatorService implements HealthIndicatorService { - - public static final String NAME = "instance_has_master"; - - private static final String INSTANCE_HAS_MASTER_GREEN_SUMMARY = "Health coordinating instance has an elected master node."; - private static final String INSTANCE_HAS_MASTER_RED_SUMMARY = "Health coordinating instance does not have an elected master node."; - - private static final String HELP_URL = "https://ela.st/fix-master"; - - private static final String NO_MASTER_INGEST_IMPACT = "The cluster cannot create, delete, or rebalance indices, and cannot insert or " - + "update documents."; - private static final String NO_MASTER_DEPLOYMENT_MANAGEMENT_IMPACT = "Scheduled tasks such as Watcher, ILM, and SLM will not work. " - + "The _cat APIs will not work."; - private static final String NO_MASTER_BACKUP_IMPACT = "Snapshot and restore will not work. Searchable snapshots cannot be mounted."; - - private final ClusterService clusterService; - - public InstanceHasMasterHealthIndicatorService(ClusterService clusterService) { - this.clusterService = clusterService; - } - - @Override - public String name() { - return NAME; - } - - @Override - public String component() { - return CLUSTER_COORDINATION; - } - - @Override - public String helpURL() { - return HELP_URL; - } - - @Override - public HealthIndicatorResult calculate(boolean explain) { - - DiscoveryNode coordinatingNode = clusterService.localNode(); - ClusterState clusterState = clusterService.state(); - DiscoveryNodes nodes = clusterState.nodes(); - DiscoveryNode masterNode = nodes.getMasterNode(); - - HealthStatus instanceHasMasterStatus = masterNode == null ? HealthStatus.RED : HealthStatus.GREEN; - String instanceHasMasterSummary = masterNode == null ? INSTANCE_HAS_MASTER_RED_SUMMARY : INSTANCE_HAS_MASTER_GREEN_SUMMARY; - List impacts = new ArrayList<>(); - if (masterNode == null) { - impacts.add(new HealthIndicatorImpact(1, NO_MASTER_INGEST_IMPACT, List.of(ImpactArea.INGEST))); - impacts.add(new HealthIndicatorImpact(1, NO_MASTER_DEPLOYMENT_MANAGEMENT_IMPACT, List.of(ImpactArea.DEPLOYMENT_MANAGEMENT))); - impacts.add(new HealthIndicatorImpact(3, NO_MASTER_BACKUP_IMPACT, List.of(ImpactArea.BACKUP))); - } - - return createIndicator(instanceHasMasterStatus, instanceHasMasterSummary, explain ? (builder, params) -> { - builder.startObject(); - builder.object("coordinating_node", xContentBuilder -> { - builder.field("node_id", coordinatingNode.getId()); - builder.field("name", coordinatingNode.getName()); - }); - builder.object("master_node", xContentBuilder -> { - if (masterNode != null) { - builder.field("node_id", masterNode.getId()); - builder.field("name", masterNode.getName()); - } else { - builder.nullField("node_id"); - builder.nullField("name"); - } - }); - return builder.endObject(); - } : HealthIndicatorDetails.EMPTY, impacts, Collections.emptyList()); - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java new file mode 100644 index 0000000000000..ccdd22ea1cd7d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java @@ -0,0 +1,461 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.coordination; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.health.HealthIndicatorDetails; +import org.elasticsearch.health.HealthIndicatorImpact; +import org.elasticsearch.health.HealthIndicatorResult; +import org.elasticsearch.health.HealthIndicatorService; +import org.elasticsearch.health.HealthStatus; +import org.elasticsearch.health.ImpactArea; +import org.elasticsearch.health.SimpleHealthIndicatorDetails; +import org.elasticsearch.health.UserAction; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.elasticsearch.health.ServerHealthComponents.CLUSTER_COORDINATION; + +/** + * This indicator reports the health of master stability. + * If we have had a master within the last 30 seconds, and that master has not changed more than 3 times in the last 30 minutes, then + * this will report GREEN. + * If we have had a master within the last 30 seconds, but that master has changed more than 3 times in the last 30 minutes (and that is + * confirmed by checking with the last-known master), then this will report YELLOW. + * If we have not had a master within the last 30 seconds, then this will will report RED with one exception. That exception is when: + * (1) no node is elected master, (2) this node is not master eligible, (3) some node is master eligible, (4) we ask a master-eligible node + * to run this indicator, and (5) it comes back with a result that is not RED. + * Since this indicator needs to be able to run when there is no master at all, it does not depend on the dedicated health node (which + * requires the existence of a master). + */ +public class StableMasterHealthIndicatorService implements HealthIndicatorService, ClusterStateListener { + + public static final String NAME = "master_is_stable"; + private static final String HELP_URL = "https://ela.st/fix-master"; + + private final ClusterService clusterService; + private final MasterHistoryService masterHistoryService; + /** + * This is the amount of time we use to make the initial decision -- have we seen a master node in the very recent past? + */ + private final TimeValue nodeHasMasterLookupTimeframe; + /** + * If the master transitions from a non-null master to a null master at least this many times it starts impacting the health status. + */ + private final int unacceptableNullTransitions; + /** + * If the master transitions from one non-null master to a different non-null master at least this many times it starts impacting the + * health status. + */ + private final int unacceptableIdentityChanges; + + private static final Logger logger = LogManager.getLogger(StableMasterHealthIndicatorService.class); + + // Keys for the details map: + private static final String DETAILS_CURRENT_MASTER = "current_master"; + private static final String DETAILS_RECENT_MASTERS = "recent_masters"; + private static final String DETAILS_EXCEPTION_FETCHING_HISTORY = "exception_fetching_history"; + + // Impacts of having an unstable master: + private static final String UNSTABLE_MASTER_INGEST_IMPACT = "The cluster cannot create, delete, or rebalance indices, and cannot " + + "insert or update documents."; + private static final String UNSTABLE_MASTER_DEPLOYMENT_MANAGEMENT_IMPACT = "Scheduled tasks such as Watcher, ILM, and SLM will not " + + "work. The _cat APIs will not work."; + private static final String UNSTABLE_MASTER_BACKUP_IMPACT = "Snapshot and restore will not work. Searchable snapshots cannot be " + + "mounted."; + + /** + * This is the list of the impacts to be reported when the master node is determined to be unstable. + */ + private static final List UNSTABLE_MASTER_IMPACTS = List.of( + new HealthIndicatorImpact(1, UNSTABLE_MASTER_INGEST_IMPACT, List.of(ImpactArea.INGEST)), + new HealthIndicatorImpact(1, UNSTABLE_MASTER_DEPLOYMENT_MANAGEMENT_IMPACT, List.of(ImpactArea.DEPLOYMENT_MANAGEMENT)), + new HealthIndicatorImpact(3, UNSTABLE_MASTER_BACKUP_IMPACT, List.of(ImpactArea.BACKUP)) + ); + + /** + * This is the default amount of time we look back to see if we have had a master at all, before moving on with other checks + */ + public static final Setting NODE_HAS_MASTER_LOOKUP_TIMEFRAME_SETTING = Setting.timeSetting( + "health.master_history.has_master_lookup_timeframe", + new TimeValue(30, TimeUnit.SECONDS), + new TimeValue(1, TimeUnit.SECONDS), + Setting.Property.NodeScope + ); + + /** + * This is the number of times that it is not OK to have a master go null. This many transitions or more will be reported as a problem. + */ + public static final Setting NO_MASTER_TRANSITIONS_THRESHOLD_SETTING = Setting.intSetting( + "health.master_history.no_master_transitions_threshold", + 4, + 0, + Setting.Property.NodeScope + ); + + /** + * This is the number of times that it is not OK to have a master change identity. This many changes or more will be reported as a + * problem. + */ + public static final Setting IDENTITY_CHANGES_THRESHOLD_SETTING = Setting.intSetting( + "health.master_history.identity_changes_threshold", + 4, + 0, + Setting.Property.NodeScope + ); + + public StableMasterHealthIndicatorService(ClusterService clusterService, MasterHistoryService masterHistoryService) { + this.clusterService = clusterService; + this.masterHistoryService = masterHistoryService; + this.nodeHasMasterLookupTimeframe = NODE_HAS_MASTER_LOOKUP_TIMEFRAME_SETTING.get(clusterService.getSettings()); + this.unacceptableNullTransitions = NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.get(clusterService.getSettings()); + this.unacceptableIdentityChanges = IDENTITY_CHANGES_THRESHOLD_SETTING.get(clusterService.getSettings()); + clusterService.addListener(this); + } + + @Override + public String name() { + return NAME; + } + + @Override + public String component() { + return CLUSTER_COORDINATION; + } + + @Override + public String helpURL() { + return HELP_URL; + } + + @Override + public HealthIndicatorResult calculate(boolean explain) { + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + if (hasSeenMasterInHasMasterLookupTimeframe()) { + return calculateOnHaveSeenMasterRecently(localMasterHistory, explain); + } else { + return calculateOnHaveNotSeenMasterRecently(localMasterHistory, explain); + } + } + + /** + * Returns the health result for the case when we have seen a master recently (at some point in the last 30 seconds). + * @param localMasterHistory The master history as seen from the local machine + * @param explain Whether to calculate and include the details and user actions in the result + * @return The HealthIndicatorResult for the given localMasterHistory + */ + private HealthIndicatorResult calculateOnHaveSeenMasterRecently(MasterHistory localMasterHistory, boolean explain) { + int masterChanges = MasterHistory.getNumberOfMasterIdentityChanges(localMasterHistory.getNodes()); + logger.trace( + "Have seen a master in the last {}): {}", + nodeHasMasterLookupTimeframe, + localMasterHistory.getMostRecentNonNullMaster() + ); + final HealthIndicatorResult result; + if (masterChanges >= unacceptableIdentityChanges) { + result = calculateOnMasterHasChangedIdentity(localMasterHistory, masterChanges, explain); + } else if (localMasterHistory.hasMasterGoneNullAtLeastNTimes(unacceptableNullTransitions)) { + result = calculateOnMasterHasFlappedNull(localMasterHistory, explain); + } else { + result = getMasterIsStableResult(explain, localMasterHistory); + } + return result; + } + + /** + * Returns the health result when we have detected locally that the master has changed identity repeatedly (by default more than 3 + * times in the last 30 minutes) + * @param localMasterHistory The master history as seen from the local machine + * @param masterChanges The number of times that the local machine has seen the master identity change in the last 30 minutes + * @param explain Whether to calculate and include the details in the result + * @return The HealthIndicatorResult for the given localMasterHistory + */ + private HealthIndicatorResult calculateOnMasterHasChangedIdentity( + MasterHistory localMasterHistory, + int masterChanges, + boolean explain + ) { + logger.trace("Have seen {} master changes in the last {}", masterChanges, localMasterHistory.getMaxHistoryAge()); + HealthStatus stableMasterStatus = HealthStatus.YELLOW; + String summary = String.format( + Locale.ROOT, + "The elected master node has changed %d times in the last %s", + masterChanges, + localMasterHistory.getMaxHistoryAge() + ); + HealthIndicatorDetails details = getDetails(explain, localMasterHistory); + List userActions = getContactSupportUserActions(explain); + return createIndicator( + stableMasterStatus, + summary, + explain ? details : HealthIndicatorDetails.EMPTY, + UNSTABLE_MASTER_IMPACTS, + userActions + ); + } + + /** + * This returns HealthIndicatorDetails.EMPTY if explain is false, otherwise a HealthIndicatorDetails object containing only a + * "current_master" object and a "recent_masters" array. The "current_master" object will have "node_id" and "name" fields for the + * master node. Both will be null if the last-seen master was null. The "recent_masters" array will contain "recent_master" objects. + * Each "recent_master" object will have "node_id" and "name" fields for the master node. These fields will never be null because + * null masters are not written to this array. + * @param explain If true, the HealthIndicatorDetails will contain "current_master" and "recent_masters". Otherwise it will be empty. + * @param localMasterHistory The MasterHistory object to pull current and recent master info from + * @return An empty HealthIndicatorDetails if explain is false, otherwise a HealthIndicatorDetails containing only "current_master" + * and "recent_masters" + */ + private HealthIndicatorDetails getDetails(boolean explain, MasterHistory localMasterHistory) { + if (explain == false) { + return HealthIndicatorDetails.EMPTY; + } + return (builder, params) -> { + builder.startObject(); + DiscoveryNode masterNode = localMasterHistory.getMostRecentMaster(); + builder.object(DETAILS_CURRENT_MASTER, xContentBuilder -> { + if (masterNode != null) { + builder.field("node_id", masterNode.getId()); + builder.field("name", masterNode.getName()); + } else { + builder.nullField("node_id"); + builder.nullField("name"); + } + }); + List recentMasters = localMasterHistory.getNodes(); + builder.array(DETAILS_RECENT_MASTERS, arrayXContentBuilder -> { + for (DiscoveryNode recentMaster : recentMasters) { + if (recentMaster != null) { + builder.startObject(); + builder.field("node_id", recentMaster.getId()); + builder.field("name", recentMaster.getName()); + builder.endObject(); + } + } + }); + return builder.endObject(); + }; + } + + /** + * This method returns the only user action that is relevant when the master is unstable -- contact support. + * @param explain If true, the returned list includes a UserAction to contact support, otherwise an empty list + * @return a single UserAction instructing users to contact support. + */ + private List getContactSupportUserActions(boolean explain) { + if (explain) { + UserAction.Definition contactSupport = new UserAction.Definition( + "contact_support", + "The Elasticsearch cluster does not have a stable master node. Please contact Elastic Support " + + "(https://support.elastic.co) to discuss available options.", + null + ); + UserAction userAction = new UserAction(contactSupport, null); + return List.of(userAction); + } else { + return List.of(); + } + } + + /** + * Returns the health result when we have detected locally that the master has changed to null repeatedly (by default more than 3 times + * in the last 30 minutes). This method attemtps to use the master history from a remote node to confirm what we are seeing locally. + * If the information from the remote node confirms that the master history has been unstable, a YELLOW status is returned. If the + * information from the remote node shows that the master history has been stable, then we assume that the problem is with this node + * and a GREEN status is returned (the problems with this node will be covered in a different health indicator). If there had been + * problems fetching the remote master history, the exception seen will be included in the details of the result. + * @param localMasterHistory The master history as seen from the local machine + * @param explain Whether to calculate and include the details in the result + * @return The HealthIndicatorResult for the given localMasterHistory + */ + private HealthIndicatorResult calculateOnMasterHasFlappedNull(MasterHistory localMasterHistory, boolean explain) { + DiscoveryNode master = localMasterHistory.getMostRecentNonNullMaster(); + boolean localNodeIsMaster = clusterService.localNode().equals(master); + List remoteHistory; + Exception remoteHistoryException = null; + if (localNodeIsMaster) { + remoteHistory = null; // We don't need to fetch the remote master's history if we are that remote master + } else { + try { + remoteHistory = masterHistoryService.getRemoteMasterHistory(); + } catch (Exception e) { + remoteHistory = null; + remoteHistoryException = e; + } + } + /* + * If the local node is master, then we have a confirmed problem (since we now know that from this node's point of view the + * master is unstable). + * If the local node is not master but the remote history is null then we have a problem (since from this node's point of view the + * master is unstable, and we were unable to get the master's own view of its history). It could just be a short-lived problem + * though if the remote history has not arrived yet. + * If the local node is not master and the master history from the master itself reports that the master has gone null repeatedly + * or changed identity repeatedly, then we have a problem (the master has confirmed what the local node saw). + */ + boolean masterConfirmedUnstable = localNodeIsMaster + || remoteHistoryException != null + || (remoteHistory != null + && (MasterHistory.hasMasterGoneNullAtLeastNTimes(remoteHistory, unacceptableNullTransitions) + || MasterHistory.getNumberOfMasterIdentityChanges(remoteHistory) >= unacceptableIdentityChanges)); + if (masterConfirmedUnstable) { + logger.trace("The master node {} thinks it is unstable", master); + String summary = String.format( + Locale.ROOT, + "The cluster's master has alternated between %s and no master multiple times in the last %s", + localMasterHistory.getNodes().stream().filter(Objects::nonNull).collect(Collectors.toSet()), + localMasterHistory.getMaxHistoryAge() + ); + final HealthIndicatorDetails details = getHealthIndicatorDetailsOnMasterHasFlappedNull( + explain, + localMasterHistory, + remoteHistoryException + ); + final List userActions = getContactSupportUserActions(explain); + return createIndicator( + HealthStatus.YELLOW, + summary, + explain ? details : HealthIndicatorDetails.EMPTY, + UNSTABLE_MASTER_IMPACTS, + userActions + ); + } else { + logger.trace("This node thinks the master is unstable, but the master node {} thinks it is stable", master); + return getMasterIsStableResult(explain, localMasterHistory); + } + } + + /** + * Returns the health indicator details for the calculateOnMasterHasFlappedNull method. The top-level objects are "current_master" and + * (optionally) "exception_fetching_history". The "current_master" object will have "node_id" and "name" fields for the master node. + * Both will be null if the last-seen master was null. + * @param explain If false, nothing is calculated and HealthIndicatorDetails.EMPTY is returned + * @param localMasterHistory The localMasterHistory + * @param remoteHistoryException An exception that was found when retrieving the remote master history. Can be null + * @return The HealthIndicatorDetails + */ + private HealthIndicatorDetails getHealthIndicatorDetailsOnMasterHasFlappedNull( + boolean explain, + MasterHistory localMasterHistory, + @Nullable Exception remoteHistoryException + ) { + if (explain == false) { + return HealthIndicatorDetails.EMPTY; + } + return (builder, params) -> { + builder.startObject(); + DiscoveryNode masterNode = localMasterHistory.getMostRecentMaster(); + builder.object(DETAILS_CURRENT_MASTER, xContentBuilder -> { + if (masterNode != null) { + builder.field("node_id", masterNode.getId()); + builder.field("name", masterNode.getName()); + } else { + builder.nullField("node_id"); + builder.nullField("name"); + } + }); + if (remoteHistoryException != null) { + builder.object(DETAILS_EXCEPTION_FETCHING_HISTORY, xContentBuilder -> { + builder.field("message", remoteHistoryException.getMessage()); + StringWriter stringWriter = new StringWriter(); + remoteHistoryException.printStackTrace(new PrintWriter(stringWriter)); + String remoteHistoryExceptionStackTrace = stringWriter.toString(); + builder.field("stack_trace", remoteHistoryExceptionStackTrace); + }); + } + return builder.endObject(); + }; + } + + /** + * Returns a HealthIndicatorResult for the case when the master is seen as stable + * @return A HealthIndicatorResult for the case when the master is seen as stable (GREEN status, no impacts or details) + */ + private HealthIndicatorResult getMasterIsStableResult(boolean explain, MasterHistory localMasterHistory) { + String summary = "The cluster has a stable master node"; + Collection impacts = new ArrayList<>(); + List userActions = List.of(); + logger.trace("The cluster has a stable master node"); + HealthIndicatorDetails details = getDetails(explain, localMasterHistory); + return createIndicator(HealthStatus.GREEN, summary, details, impacts, userActions); + } + + /** + * Returns the health result for the case when we have NOT seen a master recently (at some point in the last 30 seconds). + * @param localMasterHistory The master history as seen from the local machine + * @param explain Whether to calculate and include the details in the result + * @return The HealthIndicatorResult for the given localMasterHistory + */ + private HealthIndicatorResult calculateOnHaveNotSeenMasterRecently(MasterHistory localMasterHistory, boolean explain) { + // NOTE: The logic in this method will be implemented in a future PR + String summary = "No master has been observed recently"; + Map details = new HashMap<>(); + List userActions = getContactSupportUserActions(explain); + return createIndicator( + HealthStatus.RED, + summary, + explain ? new SimpleHealthIndicatorDetails(details) : HealthIndicatorDetails.EMPTY, + UNSTABLE_MASTER_IMPACTS, + userActions + ); + } + + /** + * This returns true if this node has seen a master node within the last few seconds + * @return true if this node has seen a master node within the last few seconds, false otherwise + */ + private boolean hasSeenMasterInHasMasterLookupTimeframe() { + // If there is currently a master, there's no point in looking at the history: + if (clusterService.state().nodes().getMasterNode() != null) { + return true; + } + return masterHistoryService.getLocalMasterHistory().hasSeenMasterInLastNSeconds((int) nodeHasMasterLookupTimeframe.seconds()); + } + + /* + * If we detect that the master has gone null 3 or more times (by default), we ask the MasterHistoryService to fetch the master + * history as seen from the most recent master node so that it is ready in case a health API request comes in. The request to the + * MasterHistoryService is made asynchronously, and populates the value that MasterHistoryService.getRemoteMasterHistory() will return. + * The remote master history is ordinarily returned very quickly if it is going to be returned, so the odds are very good it will be + * in place by the time a request for it comes in. If not, this indicator will briefly switch to yellow. + */ + @Override + public void clusterChanged(ClusterChangedEvent event) { + DiscoveryNode currentMaster = event.state().nodes().getMasterNode(); + DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode(); + if (currentMaster == null && previousMaster != null) { + if (masterHistoryService.getLocalMasterHistory().hasMasterGoneNullAtLeastNTimes(unacceptableNullTransitions)) { + DiscoveryNode master = masterHistoryService.getLocalMasterHistory().getMostRecentNonNullMaster(); + /* + * If the most recent master was this box, there is no point in making a transport request -- we already know what this + * box's view of the master history is + */ + if (master != null && clusterService.localNode().equals(master) == false) { + masterHistoryService.refreshRemoteMasterHistory(master); + } + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 0ed272d454729..7eb2d84203885 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -29,8 +29,10 @@ import org.elasticsearch.cluster.coordination.JoinValidationService; import org.elasticsearch.cluster.coordination.LagDetector; import org.elasticsearch.cluster.coordination.LeaderChecker; +import org.elasticsearch.cluster.coordination.MasterHistory; import org.elasticsearch.cluster.coordination.NoMasterBlockService; import org.elasticsearch.cluster.coordination.Reconfigurator; +import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.OperationRouting; @@ -511,6 +513,9 @@ public void apply(Settings value, Settings current, Settings previous) { IndexingPressure.MAX_INDEXING_BYTES, ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN, DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING, + StableMasterHealthIndicatorService.IDENTITY_CHANGES_THRESHOLD_SETTING, + StableMasterHealthIndicatorService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING, + MasterHistory.MAX_HISTORY_AGE_SETTING, ReadinessService.PORT ); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 8203d6ef6dee0..2fa32208c130c 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -40,8 +40,8 @@ import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.coordination.Coordinator; -import org.elasticsearch.cluster.coordination.InstanceHasMasterHealthIndicatorService; import org.elasticsearch.cluster.coordination.MasterHistoryService; +import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; import org.elasticsearch.cluster.desirednodes.DesiredNodesSettingsValidator; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -1030,7 +1030,7 @@ private HealthService createHealthService( MasterHistoryService masterHistoryService ) { List preflightHealthIndicatorServices = Collections.singletonList( - new InstanceHasMasterHealthIndicatorService(clusterService) + new StableMasterHealthIndicatorService(clusterService, masterHistoryService) ); var serverHealthIndicatorServices = List.of( new RepositoryIntegrityHealthIndicatorService(clusterService), diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java new file mode 100644 index 0000000000000..fe7f95c8917c9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java @@ -0,0 +1,525 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.coordination; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.health.HealthIndicatorDetails; +import org.elasticsearch.health.HealthIndicatorResult; +import org.elasticsearch.health.HealthStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StableMasterHealthIndicatorServiceTests extends AbstractCoordinatorTestCase { + DiscoveryNode node1; + DiscoveryNode node2; + DiscoveryNode node3; + private ClusterState nullMasterClusterState; + private ClusterState node1MasterClusterState; + private ClusterState node2MasterClusterState; + private ClusterState node3MasterClusterState; + private static final String TEST_SOURCE = "test"; + + @Before + public void setup() throws Exception { + node1 = new DiscoveryNode( + "node1", + randomNodeId(), + buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.roles(), + Version.CURRENT + ); + node2 = new DiscoveryNode( + "node2", + randomNodeId(), + buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.roles(), + Version.CURRENT + ); + node3 = new DiscoveryNode( + "node3", + randomNodeId(), + buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.roles(), + Version.CURRENT + ); + nullMasterClusterState = createClusterState(null); + node1MasterClusterState = createClusterState(node1); + node2MasterClusterState = createClusterState(node2); + node3MasterClusterState = createClusterState(node3); + } + + @SuppressWarnings("unchecked") + public void testMoreThanThreeMasterChanges() throws Exception { + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + // First master: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Null, so not counted: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Change 1: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node2MasterClusterState, nullMasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Null, so not counted: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node2MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Change 2: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Null, so not counted: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Change 3: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node3MasterClusterState, nullMasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Null, so not counted: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node3MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Still node 3, so no change: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node3MasterClusterState, nullMasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + + // Change 4: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node2MasterClusterState, node3MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.YELLOW)); + assertThat(result.summary(), equalTo("The elected master node has changed 4 times in the last 30m")); + assertThat(result.impacts().size(), equalTo(3)); + HealthIndicatorDetails details = result.details(); + Map detailsMap = xContentToMap(details); + assertThat(detailsMap.size(), equalTo(2)); + Collection recentMasters = ((Collection) detailsMap.get("recent_masters")); + // We don't show nulls in the recent_masters list: + assertThat(recentMasters.size(), equalTo(6)); + for (Object recentMaster : recentMasters) { + Map recentMasterMap = (Map) recentMaster; + assertThat(recentMasterMap.get("name"), not(emptyOrNullString())); + assertThat(recentMasterMap.get("node_id"), not(emptyOrNullString())); + } + + } + + public void testMasterGoesNull() throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node1 -> null -> node1 + * On the master node: + * node1 -> null -> node1 -> null -> node1 -> null -> node1-> null -> node1 + * In this case, the master identity changed 0 times as seen from the local node. The same master went null 4 times as seen from + * the local node. So we check the remote history. The remote history sees that the master went to null 4 times, the status is + * YELLOW. + */ + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + // Only start counting nulls once the master has been node1, so 1: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + assertThat(result.summary(), equalTo("The cluster has a stable master node")); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + // 2: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + assertThat(result.summary(), equalTo("The cluster has a stable master node")); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + // 3: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + assertThat(result.summary(), equalTo("The cluster has a stable master node")); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + // 4: + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + // It has now gone null 4 times, but the master reports that it's ok because the remote history says it has not gone null: + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + assertThat(result.summary(), equalTo("The cluster has a stable master node")); + + List sameAsLocalHistory = localMasterHistory.getNodes(); + when(masterHistoryService.getRemoteMasterHistory()).thenReturn(sameAsLocalHistory); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.YELLOW)); + assertThat(result.summary(), startsWith("The cluster's master has alternated between ")); + assertThat(result.summary(), endsWith("and no master multiple times in the last 30m")); + assertThat(result.impacts().size(), equalTo(3)); + HealthIndicatorDetails details = result.details(); + Map detailsMap = xContentToMap(details); + assertThat(detailsMap.size(), equalTo(1)); + assertThat(((Map) detailsMap.get("current_master")).get("name"), equalTo(null)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.YELLOW)); + assertThat(result.summary(), startsWith("The cluster's master has alternated between ")); + + } + + public void testMasterGoesNullWithRemoteException() throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node1 -> null -> node1 + * Connecting to the master node throws an exception + * In this case, the master identity changed 0 times as seen from the local node. The same master went null 4 times as seen from + * the local node. So we check the remote history. The remote history throws an exception, so the status is YELLOW. + */ + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + when(masterHistoryService.getRemoteMasterHistory()).thenThrow(new Exception("Failure on master")); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.YELLOW)); + assertThat(result.summary(), startsWith("The cluster's master has alternated between ")); + assertThat(result.summary(), endsWith("and no master multiple times in the last 30m")); + assertThat(result.impacts().size(), equalTo(3)); + HealthIndicatorDetails details = result.details(); + Map detailsMap = xContentToMap(details); + assertThat(detailsMap.size(), equalTo(2)); + assertThat(((Map) detailsMap.get("current_master")).get("name"), equalTo(null)); + assertThat(((Map) detailsMap.get("exception_fetching_history")).get("message"), equalTo("Failure on master")); + } + + public void testMasterGoesNullLocallyButRemotelyChangesIdentity() throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node1 -> null -> node1 + * On the master node: + * node1 -> null -> node1 -> node2 -> node3 -> node2 -> node3 + * In this case, the master identity changed 0 times as seen from the local node. The same master went null 4 times as seen from + * the local node. So we check the remote history. The master only went null here one time, but it changed identity 4 times. So we + * still get a status of YELLOW. (Note: This scenario might not be possible in the real world for a couple of reasons, but it tests + * edge cases) + */ + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + List remoteMasterHistory = new ArrayList<>(); + remoteMasterHistory.add(node1); + remoteMasterHistory.add(null); + remoteMasterHistory.add(node1); + remoteMasterHistory.add(node2); + remoteMasterHistory.add(node3); + remoteMasterHistory.add(node2); + remoteMasterHistory.add(node3); + when(masterHistoryService.getRemoteMasterHistory()).thenReturn(remoteMasterHistory); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.YELLOW)); + } + + public void testMultipleChangesButIdentityNeverChanges() throws Exception { + /* + * On the local node: + * node1 -> node1 -> node1 -> node1 -> node1 + * On the master node: + * node1 -> node1 -> node1 -> node1 -> node1 + * In this case, the master changed 4 times but there are 0 identity changes since there is only ever node1. So we never even + * check the remote master, and get a status of GREEN. (Note: This scenario is not possible in the real world because we would + * see null values in between, so it is just here to test an edge case) + */ + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + when(masterHistoryService.getRemoteMasterHistory()).thenThrow(new RuntimeException("Should never call this")); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, node1MasterClusterState)); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(HealthStatus.GREEN)); + assertThat(result.summary(), equalTo("The cluster has a stable master node")); + } + + public void testYellowOnProblematicRemoteHistory() throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node2 -> null -> node1 -> null -> node1 + * On the master node: + * node1 -> null -> node1 -> null -> node1 -> null -> node2 -> null -> node1 -> null -> node1 + * In this case we detect 2 identity changes (node1 -> node2, and node2 -> node1). We detect that node1 has gone to null 5 times. So + * we get a status of YELLOW. + */ + testTooManyTransitionsToNull(false, HealthStatus.YELLOW); + } + + public void testGreenOnNullRemoteHistory() throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node2 -> null -> node1 -> null -> node1 + * We don't get the remote master history in time so we don't know what it is. + * In this case we detect 2 identity changes (node1 -> node2, and node2 -> node1). We detect that node1 has gone to null 5 times. So + * we contact the remote master, and in this test get null in return as the master history. Since it is not definitive, we return + * GREEN. + */ + testTooManyTransitionsToNull(true, HealthStatus.GREEN); + } + + private void testTooManyTransitionsToNull(boolean remoteHistoryIsNull, HealthStatus expectedStatus) throws Exception { + /* + * On the local node: + * node1 -> null -> node1 -> null -> node1 -> null -> node2 -> null -> node1 -> null -> node1 + * On the master node: + * node1 -> null -> node1 -> null -> node1 -> null -> node2 -> null -> node1 -> null -> node1 + * In this case we detect 2 identity changes (node1 -> node2, and node2 -> node1). We detect that node1 has gone to 5 times. So + * we get a status of YELLOW. + */ + MasterHistoryService masterHistoryService = createMasterHistoryService(); + MasterHistory localMasterHistory = masterHistoryService.getLocalMasterHistory(); + StableMasterHealthIndicatorService service = createAllocationHealthIndicatorService(nullMasterClusterState, masterHistoryService); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node2MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node2MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState)); + localMasterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState)); + List remoteHistory = remoteHistoryIsNull ? null : localMasterHistory.getNodes(); + when(masterHistoryService.getRemoteMasterHistory()).thenReturn(remoteHistory); + HealthIndicatorResult result = service.calculate(true); + assertThat(result.status(), equalTo(expectedStatus)); + } + + public void testGreenForStableCluster() { + try (Cluster cluster = new Cluster(5)) { + cluster.runRandomly(); + cluster.stabilise(); + for (Cluster.ClusterNode node : cluster.clusterNodes) { + HealthIndicatorResult healthIndicatorResult = node.stableMasterHealthIndicatorService.calculate(true); + assertThat(healthIndicatorResult.status(), equalTo(HealthStatus.GREEN)); + } + } + } + + public void testRedForNoMaster() { + try (Cluster cluster = new Cluster(5, false, Settings.EMPTY)) { + cluster.runRandomly(); + cluster.stabilise(); + for (Cluster.ClusterNode node : cluster.clusterNodes) { + if (node.getLocalNode().isMasterNode()) { + node.disconnect(); + } + } + cluster.runFor(DEFAULT_STABILISATION_TIME, "Cannot call stabilise() because there is no master"); + for (Cluster.ClusterNode node : cluster.clusterNodes) { + HealthIndicatorResult healthIndicatorResult = node.stableMasterHealthIndicatorService.calculate(true); + if (node.getLocalNode().isMasterNode() == false) { + assertThat(healthIndicatorResult.status(), equalTo(HealthStatus.RED)); + } + } + while (cluster.clusterNodes.stream().anyMatch(Cluster.ClusterNode::deliverBlackholedRequests)) { + logger.debug("--> stabilising again after delivering blackholed requests"); + cluster.runFor(DEFAULT_STABILISATION_TIME, "Cannot call stabilise() because there is no master"); + } + } + } + + public void testYellowWithTooManyMasterChanges() { + testChangeMasterThreeTimes(2, 100, "The elected master node has changed"); + } + + public void testYellowWithTooManyMasterNullTransitions() { + testChangeMasterThreeTimes(100, 2, "no master multiple times"); + } + + private void testChangeMasterThreeTimes(int acceptableIdentityChanges, int acceptableNullTransitions, String expectedSummarySubstring) { + int clusterSize = 5; + int masterChanges = 3; + Settings settings = Settings.builder() + .put(StableMasterHealthIndicatorService.IDENTITY_CHANGES_THRESHOLD_SETTING.getKey(), acceptableIdentityChanges) + .put(StableMasterHealthIndicatorService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), acceptableNullTransitions) + .build(); + try (Cluster cluster = new Cluster(clusterSize, true, settings)) { + cluster.runRandomly(); + cluster.stabilise(); + + // Force the master to change by disconnecting it: + for (int i = 0; i < masterChanges; i++) { + final Cluster.ClusterNode leader = cluster.getAnyLeader(); + logger.info("--> blackholing leader {}", leader); + leader.disconnect(); + cluster.stabilise(); + } + + final Cluster.ClusterNode currentLeader = cluster.getAnyLeader(); + HealthIndicatorResult healthIndicatorResult = currentLeader.stableMasterHealthIndicatorService.calculate(true); + assertThat(healthIndicatorResult.status(), equalTo(HealthStatus.YELLOW)); + assertThat(healthIndicatorResult.summary(), containsString(expectedSummarySubstring)); + } + } + + public void testGreenAfterShrink() { + try (Cluster cluster = new Cluster(5)) { + cluster.runRandomly(); + cluster.stabilise(); + { + final Cluster.ClusterNode leader = cluster.getAnyLeader(); + logger.info("setting auto-shrink reconfiguration to false"); + leader.submitSetAutoShrinkVotingConfiguration(false); + cluster.stabilise(DEFAULT_CLUSTER_STATE_UPDATE_DELAY); + } + final Cluster.ClusterNode disconnect1 = cluster.getAnyNode(); + final Cluster.ClusterNode disconnect2 = cluster.getAnyNodeExcept(disconnect1); + + logger.info("--> disconnecting {} and {}", disconnect1, disconnect2); + disconnect1.disconnect(); + disconnect2.disconnect(); + cluster.stabilise(); + + final Cluster.ClusterNode leader = cluster.getAnyLeader(); + logger.info("setting auto-shrink reconfiguration to true"); + leader.submitSetAutoShrinkVotingConfiguration(true); + cluster.stabilise(DEFAULT_CLUSTER_STATE_UPDATE_DELAY * 2); // allow for a reconfiguration + for (Cluster.ClusterNode node : cluster.clusterNodes) { + HealthIndicatorResult healthIndicatorResult = node.stableMasterHealthIndicatorService.calculate(true); + if (leader.getLastAppliedClusterState().getLastCommittedConfiguration().getNodeIds().contains(node.getId())) { + assertThat(healthIndicatorResult.status(), equalTo(HealthStatus.GREEN)); + } + } + } + } + + private static ClusterState createClusterState(DiscoveryNode masterNode) { + var routingTableBuilder = RoutingTable.builder(); + Metadata.Builder metadataBuilder = Metadata.builder(); + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + if (masterNode != null) { + nodesBuilder.masterNodeId(masterNode.getId()); + nodesBuilder.add(masterNode); + } + return ClusterState.builder(new ClusterName("test-cluster")) + .routingTable(routingTableBuilder.build()) + .metadata(metadataBuilder.build()) + .nodes(nodesBuilder) + .build(); + } + + private static String randomNodeId() { + return UUID.randomUUID().toString(); + } + + /* + * Creates a mocked MasterHistoryService with a non-mocked local master history (which can be updated with clusterChanged calls). The + * remote master history is mocked. + */ + private static MasterHistoryService createMasterHistoryService() throws Exception { + var clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); + MasterHistory localMasterHistory = new MasterHistory(threadPool, clusterService); + MasterHistoryService masterHistoryService = mock(MasterHistoryService.class); + when(masterHistoryService.getLocalMasterHistory()).thenReturn(localMasterHistory); + List remoteMasterHistory = new ArrayList<>(); + when(masterHistoryService.getRemoteMasterHistory()).thenReturn(remoteMasterHistory); + return masterHistoryService; + } + + private static StableMasterHealthIndicatorService createAllocationHealthIndicatorService( + ClusterState clusterState, + MasterHistoryService masterHistoryService + ) { + var clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + when(clusterService.state()).thenReturn(clusterState); + DiscoveryNode localNode = mock(DiscoveryNode.class); + when(clusterService.localNode()).thenReturn(localNode); + when(localNode.isMasterNode()).thenReturn(false); + Coordinator coordinator = mock(Coordinator.class); + when(coordinator.getFoundPeers()).thenReturn(Collections.emptyList()); + return new StableMasterHealthIndicatorService(clusterService, masterHistoryService); + } + + private Map xContentToMap(ToXContent xcontent) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + xcontent.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput()); + return parser.map(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java index 13cf2d41509a7..43c805b6f18b3 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -15,6 +15,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.coordination.MasterHistoryAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.support.ActionFilters; @@ -109,7 +110,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; -import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.DEFAULT_DELAY_VARIABILITY; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.BOOTSTRAP_PLACEHOLDER_PREFIX; import static org.elasticsearch.cluster.coordination.CoordinationStateTestCluster.clusterState; @@ -1084,6 +1084,8 @@ public class ClusterNode { private DisruptableClusterApplierService clusterApplierService; private ClusterService clusterService; TransportService transportService; + private MasterHistoryService masterHistoryService; + StableMasterHealthIndicatorService stableMasterHealthIndicatorService; private DisruptableMockTransport mockTransport; private final NodeHealthService nodeHealthService; List> extraJoinValidators = new ArrayList<>(); @@ -1219,6 +1221,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { threadPool ); clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService); + masterHistoryService = new MasterHistoryService(transportService, threadPool, clusterService); + stableMasterHealthIndicatorService = new StableMasterHealthIndicatorService(clusterService, masterHistoryService); clusterService.setNodeConnectionsService( new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService) ); @@ -1228,9 +1232,11 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { final AllocationService allocationService = ESAllocationTestCase.createAllocationService(Settings.EMPTY); final NodeClient client = new NodeClient(Settings.EMPTY, threadPool); client.initialize( - singletonMap( + Map.of( NodesHotThreadsAction.INSTANCE, - new TransportNodesHotThreadsAction(threadPool, clusterService, transportService, new ActionFilters(emptySet())) + new TransportNodesHotThreadsAction(threadPool, clusterService, transportService, new ActionFilters(emptySet())), + MasterHistoryAction.INSTANCE, + new MasterHistoryAction.TransportAction(transportService, new ActionFilters(Set.of()), masterHistoryService) ), transportService.getTaskManager(), localNode::getId,