[Zen2] Introduce vote withdrawal (#35446)

If shutting down half or more of the master-eligible nodes, their votes must
first be explicitly withdrawn to ensure that the cluster doesn't lose its
quorum. This works via _voting tombstones_, stored in the cluster state, which
tell the reconfigurator to remove nodes from the voting configuration.

This change introduces voting tombstones to the cluster state, together with
transport APIs for adding and removing them, and makes use of these APIs in
`InternalTestCluster` to support tests which remove at least half of the
master-eligible nodes at once (e.g. shrinking from two master-eligible nodes to
one).

Author: DaveCTurner

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -27,6 +27,10 @@
 import org.elasticsearch.action.admin.cluster.bootstrap.GetDiscoveredNodesAction;
 import org.elasticsearch.action.admin.cluster.bootstrap.TransportBootstrapClusterAction;
 import org.elasticsearch.action.admin.cluster.bootstrap.TransportGetDiscoveredNodesAction;
+import org.elasticsearch.action.admin.cluster.configuration.AddVotingTombstonesAction;
+import org.elasticsearch.action.admin.cluster.configuration.ClearVotingTombstonesAction;
+import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingTombstonesAction;
+import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingTombstonesAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction;
@@ -428,6 +432,8 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
 
         actions.register(GetDiscoveredNodesAction.INSTANCE, TransportGetDiscoveredNodesAction.class);
         actions.register(BootstrapClusterAction.INSTANCE, TransportBootstrapClusterAction.class);
+        actions.register(AddVotingTombstonesAction.INSTANCE, TransportAddVotingTombstonesAction.class);
+        actions.register(ClearVotingTombstonesAction.INSTANCE, TransportClearVotingTombstonesAction.class);
         actions.register(ClusterAllocationExplainAction.INSTANCE, TransportClusterAllocationExplainAction.class);
         actions.register(ClusterStatsAction.INSTANCE, TransportClusterStatsAction.class);
         actions.register(ClusterStateAction.INSTANCE, TransportClusterStateAction.class);

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingTombstonesAction.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.configuration;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+public class AddVotingTombstonesAction extends Action<AddVotingTombstonesResponse> {
+    public static final AddVotingTombstonesAction INSTANCE = new AddVotingTombstonesAction();
+    public static final String NAME = "cluster:admin/voting/add_tombstones";
+
+    private AddVotingTombstonesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public AddVotingTombstonesResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Reader<AddVotingTombstonesResponse> getResponseReader() {
+        return AddVotingTombstonesResponse::new;
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingTombstonesRequest.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.configuration;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A request to add voting tombstones for certain master-eligible nodes, and wait for these nodes to be removed from the voting
+ * configuration.
+ */
+public class AddVotingTombstonesRequest extends MasterNodeRequest<AddVotingTombstonesRequest> {
+    private final String[] nodeDescriptions;
+    private final TimeValue timeout;
+
+    /**
+     * Construct a request to add voting tombstones for master-eligible nodes matching the given descriptions, and wait for a default 30
+     * seconds for these nodes to be removed from the voting configuration.
+     * @param nodeDescriptions Descriptions of the nodes to add - see {@link DiscoveryNodes#resolveNodes(String...)}
+     */
+    public AddVotingTombstonesRequest(String[] nodeDescriptions) {
+        this(nodeDescriptions, TimeValue.timeValueSeconds(30));
+    }
+
+    /**
+     * Construct a request to add voting tombstones for master-eligible nodes matching the given descriptions, and wait for these nodes to
+     * be removed from the voting configuration.
+     * @param nodeDescriptions Descriptions of the nodes whose tombstones to add - see {@link DiscoveryNodes#resolveNodes(String...)}.
+     * @param timeout How long to wait for the nodes to be removed from the voting configuration.
+     */
+    public AddVotingTombstonesRequest(String[] nodeDescriptions, TimeValue timeout) {
+        if (timeout.compareTo(TimeValue.ZERO) < 0) {
+            throw new IllegalArgumentException("timeout [" + timeout + "] must be non-negative");
+        }
+        this.nodeDescriptions = nodeDescriptions;
+        this.timeout = timeout;
+    }
+
+    public AddVotingTombstonesRequest(StreamInput in) throws IOException {
+        super(in);
+        nodeDescriptions = in.readStringArray();
+        timeout = in.readTimeValue();
+    }
+
+    Set<DiscoveryNode> resolveNodes(ClusterState currentState) {
+        final DiscoveryNodes allNodes = currentState.nodes();
+        final Set<DiscoveryNode> resolvedNodes = Arrays.stream(allNodes.resolveNodes(nodeDescriptions))
+            .map(allNodes::get).filter(DiscoveryNode::isMasterNode).collect(Collectors.toSet());
+
+        if (resolvedNodes.isEmpty()) {
+            throw new IllegalArgumentException("add voting tombstones request for " + Arrays.asList(nodeDescriptions)
+                + " matched no master-eligible nodes");
+        }
+
+        resolvedNodes.removeIf(n -> currentState.getVotingTombstones().contains(n));
+        return resolvedNodes;
+    }
+
+    Set<DiscoveryNode> resolveNodesAndCheckMaximum(ClusterState currentState, int maxTombstoneCount, String maximumSettingKey) {
+        final Set<DiscoveryNode> resolvedNodes = resolveNodes(currentState);
+
+        final int oldTombstoneCount = currentState.getVotingTombstones().size();
+        final int newTombstoneCount = resolvedNodes.size();
+        if (oldTombstoneCount + newTombstoneCount > maxTombstoneCount) {
+            throw new IllegalArgumentException("add voting tombstones request for " + Arrays.asList(nodeDescriptions)
+                + " would add [" + newTombstoneCount + "] voting tombstones to the existing [" + oldTombstoneCount
+                + "] which would exceed the maximum of [" + maxTombstoneCount + "] set by ["
+                + maximumSettingKey + "]");
+        }
+        return resolvedNodes;
+    }
+
+    /**
+     * @return descriptions of the nodes for whom to add tombstones.
+     */
+    public String[] getNodeDescriptions() {
+        return nodeDescriptions;
+    }
+
+    /**
+     * @return how long to wait after adding the tombstones for the nodes to be removed from the voting configuration.
+     */
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(nodeDescriptions);
+        out.writeTimeValue(timeout);
+    }
+
+    @Override
+    public String toString() {
+        return "AddVotingTombstonesRequest{" +
+            "nodeDescriptions=" + Arrays.asList(nodeDescriptions) +
+            ", timeout=" + timeout +
+            '}';
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingTombstonesResponse.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.configuration;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * A response to {@link AddVotingTombstonesRequest} indicating that voting tombstones have been added for the requested nodes and these
+ * nodes have been removed from the voting configuration.
+ */
+public class AddVotingTombstonesResponse extends ActionResponse {
+
+    public AddVotingTombstonesResponse() {
+    }
+
+    public AddVotingTombstonesResponse(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingTombstonesAction.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.configuration;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+public class ClearVotingTombstonesAction extends Action<ClearVotingTombstonesResponse> {
+    public static final ClearVotingTombstonesAction INSTANCE = new ClearVotingTombstonesAction();
+    public static final String NAME = "cluster:admin/voting/clear_tombstones";
+
+    private ClearVotingTombstonesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public ClearVotingTombstonesResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Reader<ClearVotingTombstonesResponse> getResponseReader() {
+        return ClearVotingTombstonesResponse::new;
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingTombstonesRequest.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.configuration;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+
+/**
+ * A request to clear the voting tombstones from the cluster state, optionally waiting for these nodes to be removed from the cluster first.
+ */
+public class ClearVotingTombstonesRequest extends MasterNodeRequest<ClearVotingTombstonesRequest> {
+    private boolean waitForRemoval = true;
+    private TimeValue timeout = TimeValue.timeValueSeconds(30);
+
+    /**
+     * Construct a request to remove all the voting tombstones from the cluster state.
+     */
+    public ClearVotingTombstonesRequest() {
+    }
+
+    public ClearVotingTombstonesRequest(StreamInput in) throws IOException {
+        super(in);
+        waitForRemoval = in.readBoolean();
+        timeout = in.readTimeValue();
+    }
+
+    /**
+     * @return whether to wait for the tombstoned nodes to be removed from the cluster before removing their tombstones. True by default.
+     */
+    public boolean getWaitForRemoval() {
+        return waitForRemoval;
+    }
+
+    /**
+     * @param waitForRemoval whether to wait for the tombstoned nodes to be removed from the cluster before removing their tombstones. True
+     *                       by default.
+     */
+    public void setWaitForRemoval(boolean waitForRemoval) {
+        this.waitForRemoval = waitForRemoval;
+    }
+
+    /**
+     * @param timeout how long to wait for the tombstoned nodes to be removed if {@link ClearVotingTombstonesRequest#waitForRemoval} is
+     *                true. Defaults to 30 seconds.
+     */
+    public void setTimeout(TimeValue timeout) {
+        this.timeout = timeout;
+    }
+
+    /**
+     * @return how long to wait for the tombstoned nodes to be removed if {@link ClearVotingTombstonesRequest#waitForRemoval} is
+     * true. Defaults to 30 seconds.
+     */
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(waitForRemoval);
+        out.writeTimeValue(timeout);
+    }
+
+    @Override
+    public String toString() {
+        return "ClearVotingTombstonesRequest{" +
+            ", waitForRemoval=" + waitForRemoval +
+            ", timeout=" + timeout +
+            '}';
+    }
+}