Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test the Cluster Shard Allocation Explain API with closed indices #38631

Merged
merged 5 commits into from
Feb 18, 2019
Merged

Test the Cluster Shard Allocation Explain API with closed indices #38631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bf13c88
Adapt ClusterAllocationExplainIT to also test closed indices
tlrx Feb 8, 2019
df139d6
Apply feedback
tlrx Feb 9, 2019
5484f6d
Add setWaitForNoInitializingShards
tlrx Feb 11, 2019
5c1a95a
Merge branch 'replicated-closed-indices' into rci-ClusterAllocationEx…
tlrx Feb 11, 2019
f1dcee8
Merge branch 'replicated-closed-indices' into rci-ClusterAllocationEx…
tlrx Feb 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,47 @@
- match: { primary: false }
- is_true: cluster_info
- is_true: can_allocate


---
"Cluster shard allocation explanation test with a closed index":
- skip:
version: " - 7.99.99"
reason: closed indices are replicated starting version 8.0.0

- do:
indices.create:
index: test_closed
body: { "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 } }

- match: { acknowledged: true }

- do:
cluster.health:
index: test_closed
wait_for_status: green

- do:
indices.close:
index: test_closed

- match: { acknowledged: true }

- do:
cluster.health:
index: test_closed
wait_for_status: green

- do:
cluster.allocation_explain:
body: { "index": "test_closed", "shard": 0, "primary": true }

- match: { current_state: "started" }
- is_true: current_node.id
- match: { index: "test_closed" }
- match: { shard: 0 }
- match: { primary: true }
- is_true: can_remain_on_current_node
- is_true: can_rebalance_cluster
- is_true: can_rebalance_to_other_node
- is_true: rebalance_explanation
83 changes: 57 additions & 26 deletions ...st/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@

package org.elasticsearch.action.admin.cluster.allocation;

import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.UnassignedInfo;
Expand All @@ -32,8 +35,10 @@
import org.elasticsearch.cluster.routing.allocation.MoveDecision;
import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -52,6 +57,7 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand All @@ -70,8 +76,7 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception {
logger.info("--> starting 2 nodes");
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary, 0 replicas");
createIndexAndIndexData(1, 0);
prepareIndex(1, 0);

logger.info("--> stopping the node with the primary");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
Expand Down Expand Up @@ -149,8 +154,7 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {
logger.info("--> starting 3 nodes");
internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary, 1 replica");
createIndexAndIndexData(1, 1);
prepareIndex(1, 1);
logger.info("--> stopping the node with the replica");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName()));
ensureStableCluster(2);
Expand Down Expand Up @@ -268,8 +272,7 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception {
logger.info("--> starting 3 nodes");
List<String> nodes = internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary and 1 replica");
createIndexAndIndexData(1, 1);
prepareIndex(1, 1);
String primaryNodeName = primaryNodeName();
nodes.remove(primaryNodeName);

Expand Down Expand Up @@ -390,7 +393,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
prepareIndex(IndexMetaData.State.OPEN, 1, 0,
Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
ActiveShardCount.NONE);

boolean includeYesDecisions = randomBoolean();
Expand Down Expand Up @@ -481,8 +485,7 @@ public void testAllocationFilteringPreventsShardMove() throws Exception {
logger.info("--> starting 2 nodes");
internalCluster().startNodes(2);

logger.info("--> creating an index with 1 primary and 0 replicas");
createIndexAndIndexData(1, 0);
prepareIndex(1, 0);

logger.info("--> setting up allocation filtering to prevent allocation to both nodes");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -591,8 +594,7 @@ public void testRebalancingNotAllowed() throws Exception {
internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(5, 0);

logger.info("--> disabling rebalancing on the index");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -704,8 +706,7 @@ public void testWorseBalance() throws Exception {
internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(5, 0);

logger.info("--> setting balancing threshold really high, so it won't be met");
client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Expand Down Expand Up @@ -808,8 +809,7 @@ public void testBetterBalanceButCannotAllocate() throws Exception {
String firstNode = internalCluster().startNode();
ensureStableCluster(1);

logger.info("--> creating an index with 5 shards, all allocated to the single node");
createIndexAndIndexData(5, 0);
prepareIndex(5, 0);

logger.info("--> setting up allocation filtering to only allow allocation to the current node");
client().admin().indices().prepareUpdateSettings("idx").setSettings(
Expand Down Expand Up @@ -918,9 +918,9 @@ public void testAssignedReplicaOnSpecificNode() throws Exception {
logger.info("--> starting 3 nodes");
List<String> nodes = internalCluster().startNodes(3);

logger.info("--> creating an index with 1 primary and 2 replicas");
String excludedNode = nodes.get(randomIntBetween(0, 2));
createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
prepareIndex(randomIndexState(), 1, 2,
Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
ActiveShardCount.from(2));

boolean includeYesDecisions = randomBoolean();
Expand Down Expand Up @@ -1019,8 +1019,7 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
final String replicaNode = internalCluster().startNode();
final String primaryNode = internalCluster().startNode();

logger.info("--> creating an index with 1 primary and 1 replica");
createIndexAndIndexData(1, 1,
prepareIndex(IndexMetaData.State.OPEN, 1, 1,
Settings.builder()
.put("index.routing.allocation.include._name", primaryNode)
.put("index.routing.allocation.exclude._name", masterNode)
Expand All @@ -1037,8 +1036,22 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
logger.info("--> stop node with the replica shard");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));

logger.info("--> index more data, now the replica is stale");
indexData();
final IndexMetaData.State indexState = randomIndexState();
if (indexState == IndexMetaData.State.OPEN) {
logger.info("--> index more data, now the replica is stale");
indexData();
} else {
logger.info("--> close the index, now the replica is stale");
assertAcked(client().admin().indices().prepareClose("idx"));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will this turn cluster health RED? If so, I think that that is not desired behavior. Let's add a bullet point to the meta-issue that we will change that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does not turn the cluster health red. Here we use the cluster health to check for index health after the close so that there's no initializing shards. I've added a missing setWaitForNoInitializingShards.


final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
.setTimeout(TimeValue.timeValueSeconds(30))
.setWaitForActiveShards(ActiveShardCount.ONE)
.setWaitForNoInitializingShards(true)
.setWaitForEvents(Priority.LANGUID)
.get();
assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
}

logger.info("--> stop the node with the primary");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
Expand Down Expand Up @@ -1147,21 +1160,39 @@ private ClusterAllocationExplanation runExplain(boolean primary, String nodeId,
return explanation;
}

private void createIndexAndIndexData(int numPrimaries, int numReplicas) {
createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
private void prepareIndex(final int numPrimaries, final int numReplicas) {
prepareIndex(randomIndexState(), numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
}

private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) {
client().admin().indices().prepareCreate("idx")
private void prepareIndex(final IndexMetaData.State state, final int numPrimaries, final int numReplicas,
final Settings settings, final ActiveShardCount activeShardCount) {

logger.info("--> creating a {} index with {} primary, {} replicas", state, numPrimaries, numReplicas);
assertAcked(client().admin().indices().prepareCreate("idx")
.setSettings(Settings.builder()
.put("index.number_of_shards", numPrimaries)
.put("index.number_of_replicas", numReplicas)
.put(settings))
.setWaitForActiveShards(activeShardCount)
.get();
.get());

if (activeShardCount != ActiveShardCount.NONE) {
indexData();
}
if (state == IndexMetaData.State.CLOSE) {
assertAcked(client().admin().indices().prepareClose("idx"));

final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
.setTimeout(TimeValue.timeValueSeconds(30))
.setWaitForActiveShards(activeShardCount)
.setWaitForEvents(Priority.LANGUID)
.get();
assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
}
}

private static IndexMetaData.State randomIndexState() {
return randomFrom(IndexMetaData.State.values());
}

private void indexData() {
Expand Down