Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test the Cluster Shard Allocation Explain API with closed indices #38631

Merged
merged 5 commits into from
Feb 18, 2019
Merged

Test the Cluster Shard Allocation Explain API with closed indices #38631

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bf13c88
Adapt ClusterAllocationExplainIT to also test closed indices
tlrx Feb 8, 2019
df139d6
Apply feedback
tlrx Feb 9, 2019
5484f6d
Add setWaitForNoInitializingShards
tlrx Feb 11, 2019
5c1a95a
Merge branch 'replicated-closed-indices' into rci-ClusterAllocationEx…
tlrx Feb 11, 2019
f1dcee8
Merge branch 'replicated-closed-indices' into rci-ClusterAllocationEx…
tlrx Feb 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,47 @@
- match: { primary: false }
- is_true: cluster_info
- is_true: can_allocate


---
"Cluster shard allocation explanation test with a closed index":
- skip:
version: " - 7.99.99"
reason: closed indices are replicated starting version 8.0.0

- do:
indices.create:
index: test_closed
body: { "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 } }

- match: { acknowledged: true }

- do:
cluster.health:
index: test_closed
wait_for_status: green

- do:
indices.close:
index: test_closed

- match: { acknowledged: true }

- do:
cluster.health:
index: test_closed
wait_for_status: green

- do:
cluster.allocation_explain:
body: { "index": "test_closed", "shard": 0, "primary": true }

- match: { current_state: "started" }
- is_true: current_node.id
- match: { index: "test_closed" }
- match: { shard: 0 }
- match: { primary: true }
- is_true: can_remain_on_current_node
- is_true: can_rebalance_cluster
- is_true: can_rebalance_to_other_node
- is_true: rebalance_explanation
168 changes: 132 additions & 36 deletions ...st/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@

package org.elasticsearch.action.admin.cluster.allocation;

import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.UnassignedInfo;
Expand All @@ -32,8 +35,10 @@
import org.elasticsearch.cluster.routing.allocation.MoveDecision;
import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -52,6 +57,7 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand All @@ -66,12 +72,19 @@
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
public final class ClusterAllocationExplainIT extends ESIntegTestCase {

public void testUnassignedPrimaryWithExistingIndex() throws Exception {
public void testUnassignedPrimaryWithExistingOpenedIndex() throws Exception {
runTestUnassignedPrimaryWithExistingIndex(IndexMetaData.State.OPEN);
}

public void testUnassignedPrimaryWithExistingClosedIndex() throws Exception {
tlrx marked this conversation as resolved.
Show resolved Hide resolved
runTestUnassignedPrimaryWithExistingIndex(IndexMetaData.State.CLOSE);
}

private void runTestUnassignedPrimaryWithExistingIndex(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 2 nodes");
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary, 0 replicas");
createIndexAndIndexData(1, 0);
prepareIndex(indexState, 1, 0);

logger.info("--> stopping the node with the primary");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
Expand Down Expand Up @@ -145,12 +158,19 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception {
}
}

public void testUnassignedReplicaDelayedAllocation() throws Exception {
public void testUnassignedReplicaDelayedAllocationForOpenedIndex() throws Exception {
runTestUnassignedReplicaDelayedAllocation(IndexMetaData.State.OPEN);
}

public void testUnassignedReplicaDelayedAllocationForClosedIndex() throws Exception {
runTestUnassignedReplicaDelayedAllocation(IndexMetaData.State.CLOSE);
}

private void runTestUnassignedReplicaDelayedAllocation(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 3 nodes");
internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary, 1 replica");
createIndexAndIndexData(1, 1);
prepareIndex(indexState, 1, 1);
logger.info("--> stopping the node with the replica");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName()));
ensureStableCluster(2);
Expand Down Expand Up @@ -264,12 +284,19 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {
}
}

public void testUnassignedReplicaWithPriorCopy() throws Exception {
public void testUnassignedReplicaWithPriorCopyForOpenedIndex() throws Exception {
runTestUnassignedReplicaWithPriorCopy(IndexMetaData.State.OPEN);
}

public void testUnassignedReplicaWithPriorCopyForClosedIndex() throws Exception {
runTestUnassignedReplicaWithPriorCopy(IndexMetaData.State.CLOSE);
}

private void runTestUnassignedReplicaWithPriorCopy(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 3 nodes");
List<String> nodes = internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary and 1 replica");
createIndexAndIndexData(1, 1);
prepareIndex(indexState, 1, 1);
String primaryNodeName = primaryNodeName();
nodes.remove(primaryNodeName);

Expand Down Expand Up @@ -390,8 +417,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
ActiveShardCount.NONE);
prepareIndex(IndexMetaData.State.OPEN, 1, 0,
Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(), ActiveShardCount.NONE);

boolean includeYesDecisions = randomBoolean();
boolean includeDiskInfo = randomBoolean();
Expand Down Expand Up @@ -477,12 +504,19 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {
}
}

public void testAllocationFilteringPreventsShardMove() throws Exception {
public void testAllocationFilteringPreventsShardMoveForOpenedIndex() throws Exception {
runTestAllocationFilteringPreventsShardMove(IndexMetaData.State.OPEN);
}

public void testAllocationFilteringPreventsShardMoveForClosedIndex() throws Exception {
runTestAllocationFilteringPreventsShardMove(IndexMetaData.State.CLOSE);
}

private void runTestAllocationFilteringPreventsShardMove(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 2 nodes");
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary and 0 replicas");
createIndexAndIndexData(1, 0);
prepareIndex(indexState, 1, 0);

logger.info("--> setting up allocation filtering to prevent allocation to both nodes");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -586,13 +620,21 @@ public void testAllocationFilteringPreventsShardMove() throws Exception {
}
}

public void testRebalancingNotAllowed() throws Exception {
public void testRebalancingNotAllowedForOpenedIndex() throws Exception {
runTestRebalancingNotAllowed(IndexMetaData.State.OPEN);
}

public void testRebalancingNotAllowedForClosedIndex() throws Exception {
runTestRebalancingNotAllowed(IndexMetaData.State.CLOSE);
}

private void runTestRebalancingNotAllowed(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting a single node");
internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(indexState, 5, 0);

logger.info("--> disabling rebalancing on the index");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -699,13 +741,20 @@ public void testRebalancingNotAllowed() throws Exception {
}
}

public void testWorseBalance() throws Exception {
public void testWorseBalanceForOpenedIndex() throws Exception {
runTestWorseBalance(IndexMetaData.State.OPEN);
}

public void testWorseBalanceForClosedIndex() throws Exception {
runTestWorseBalance(IndexMetaData.State.CLOSE);
}

private void runTestWorseBalance(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting a single node");
internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(indexState, 5, 0);

logger.info("--> setting balancing threshold really high, so it won't be met");
client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Expand Down Expand Up @@ -803,13 +852,20 @@ public void testWorseBalance() throws Exception {
}
}

public void testBetterBalanceButCannotAllocate() throws Exception {
public void testBetterBalanceButCannotAllocateForOpenedIndex() throws Exception {
runTestBetterBalanceButCannotAllocate(IndexMetaData.State.OPEN);
}

public void testBetterBalanceButCannotAllocateForClosedIndex() throws Exception {
runTestBetterBalanceButCannotAllocate(IndexMetaData.State.CLOSE);
}

private void runTestBetterBalanceButCannotAllocate(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting a single node");
String firstNode = internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(indexState, 5, 0);

logger.info("--> setting up allocation filtering to only allow allocation to the current node");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -914,14 +970,21 @@ public void testBetterBalanceButCannotAllocate() throws Exception {
}
}

public void testAssignedReplicaOnSpecificNode() throws Exception {
public void testAssignedReplicaOnSpecificNodeForOpenedIndex() throws Exception {
runTestAssignedReplicaOnSpecificNode(IndexMetaData.State.OPEN);
}

public void testAssignedReplicaOnSpecificNodeForClosedIndex() throws Exception {
runTestAssignedReplicaOnSpecificNode(IndexMetaData.State.CLOSE);
}

private void runTestAssignedReplicaOnSpecificNode(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 3 nodes");
List<String> nodes = internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary and 2 replicas");
String excludedNode = nodes.get(randomIntBetween(0, 2));
createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
ActiveShardCount.from(2));
prepareIndex(indexState, 1, 2,
Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(), ActiveShardCount.from(2));

boolean includeYesDecisions = randomBoolean();
boolean includeDiskInfo = randomBoolean();
Expand Down Expand Up @@ -1011,16 +1074,23 @@ public void testAssignedReplicaOnSpecificNode() throws Exception {
}
}

public void testCannotAllocateStaleReplicaExplanation() throws Exception {
public void testCannotAllocateStaleReplicaExplanationForOpenedIndex() throws Exception {
runTestCannotAllocateStaleReplicaExplanation(IndexMetaData.State.OPEN);
}

public void testCannotAllocateStaleReplicaExplanationForClosedIndex() throws Exception {
runTestCannotAllocateStaleReplicaExplanation(IndexMetaData.State.CLOSE);
}

private void runTestCannotAllocateStaleReplicaExplanation(final IndexMetaData.State indexState) throws Exception {
logger.info("--> starting 3 nodes");
final String masterNode = internalCluster().startNode();
// start replica node first, so it's path will be used first when we start a node after
// stopping all of them at end of test.
final String replicaNode = internalCluster().startNode();
final String primaryNode = internalCluster().startNode();

logger.info("--> creating an index with 1 primary and 1 replica");
createIndexAndIndexData(1, 1,
prepareIndex(IndexMetaData.State.OPEN, 1, 1,
Settings.builder()
.put("index.routing.allocation.include._name", primaryNode)
.put("index.routing.allocation.exclude._name", masterNode)
Expand All @@ -1037,8 +1107,20 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
logger.info("--> stop node with the replica shard");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));

logger.info("--> index more data, now the replica is stale");
indexData();
if (indexState == IndexMetaData.State.OPEN) {
logger.info("--> index more data, now the replica is stale");
indexData();
} else {
logger.info("--> close the index, now the replica is stale");
assertAcked(client().admin().indices().prepareClose("idx"));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will this turn cluster health RED? If so, I think that that is not desired behavior. Let's add a bullet point to the meta-issue that we will change that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does not turn the cluster health red. Here we use the cluster health to check for index health after the close so that there's no initializing shards. I've added a missing setWaitForNoInitializingShards.


final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
.setTimeout(TimeValue.timeValueSeconds(30))
.setWaitForActiveShards(ActiveShardCount.ONE)
.setWaitForEvents(Priority.LANGUID)
.get();
assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
}

logger.info("--> stop the node with the primary");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
Expand Down Expand Up @@ -1147,21 +1229,35 @@ private ClusterAllocationExplanation runExplain(boolean primary, String nodeId,
return explanation;
}

private void createIndexAndIndexData(int numPrimaries, int numReplicas) {
createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
private void prepareIndex(final IndexMetaData.State state, final int numPrimaries, final int numReplicas) {
prepareIndex(state, numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
}

private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) {
client().admin().indices().prepareCreate("idx")
private void prepareIndex(final IndexMetaData.State state, final int numPrimaries, final int numReplicas,
final Settings settings, final ActiveShardCount activeShardCount) {

logger.info("--> creating a {} index with {} primary, {} replicas", state, numPrimaries, numReplicas);
assertAcked(client().admin().indices().prepareCreate("idx")
.setSettings(Settings.builder()
.put("index.number_of_shards", numPrimaries)
.put("index.number_of_replicas", numReplicas)
.put(settings))
.setWaitForActiveShards(activeShardCount)
.get();
.get());

if (activeShardCount != ActiveShardCount.NONE) {
indexData();
}
if (state == IndexMetaData.State.CLOSE) {
assertAcked(client().admin().indices().prepareClose("idx"));

final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
.setTimeout(TimeValue.timeValueSeconds(30))
.setWaitForActiveShards(activeShardCount)
.setWaitForEvents(Priority.LANGUID)
.get();
assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
}
}

private void indexData() {
Expand Down