From 6a7757ac95134f3575dc212e36a5e59d74a28f3e Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 5 Feb 2019 14:26:06 +0100 Subject: [PATCH 1/4] Expose Recovery API for closed indices --- .../test/cat.recovery/10_basic.yml | 56 +++++++++++++++++++ .../test/indices.recovery/10_basic.yml | 50 +++++++++++++++++ .../indices/recovery/RecoveryRequest.java | 3 +- .../recovery/TransportRecoveryAction.java | 4 +- .../support/broadcast/BroadcastRequest.java | 5 ++ .../indices/recovery/IndexRecoveryIT.java | 56 +++++++++++++------ 6 files changed, 155 insertions(+), 19 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index 69ceccc1ef3bf..f635932c34433 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -75,3 +75,59 @@ \n )+ $/ + +--- +"Test cat recovery output for closed index": + - skip: + version: " - 7.99.99" + reason: closed indices are replicated starting version 8.0.0 + + - do: + index: + index: index2 + id: 1 + body: { foo: bar } + refresh: true + + - do: + indices.close: + index: index2 + - is_true: acknowledged + + - do: + cluster.health: + index: index2 + wait_for_status: yellow + + - do: + cat.recovery: + index: index2 + h: i,s,t,ty,st,shost,thost,rep,snap,f,fr,fp,tf,b,br,bp,tb,to,tor,top + + - match: + $body: | + /^ + ( + index2 \s+ + \d \s+ # shard + (?:\d+ms|\d+(?:\.\d+)?s) \s+ # time in ms or seconds + existing_store \s+ # source type (always existing_store for closed indices) + done \s+ # stage + [-\w./]+ \s+ # source_host + [-\w./]+ \s+ # target_host + [-\w./]+ \s+ # repository + [-\w./]+ \s+ # snapshot + \d+ \s+ # files + \d+ \s+ # files_recovered + \d+\.\d+% \s+ # files_percent + \d+ \s+ # files_total + \d+ \s+ # bytes + \d+ \s+ # bytes_recovered + \d+\.\d+% \s+ # bytes_percent + \d+ \s+ # bytes_total + -1 \s+ # translog_ops (always -1 for closed indices) + 0 \s+ # translog_ops_recovered (always 0 for closed indices) + -1\.0% # translog_ops_percent (always -1.0% for closed indices) + \n + )+ + $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml index fd8937a23cdee..c5df3b053583f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml @@ -40,6 +40,56 @@ - gte: { test_1.shards.0.verify_index.check_index_time_in_millis: 0 } - gte: { test_1.shards.0.verify_index.total_time_in_millis: 0 } --- +"Indices recovery test for closed index": + - skip: + version: " - 7.99.99" + reason: closed indices are replicated starting version 8.0.0 + + - do: + indices.create: + index: test_2 + body: + settings: + index: + number_of_replicas: 0 + + - do: + indices.close: + index: test_2 + - is_true: acknowledged + + - do: + cluster.health: + index: test_2 + wait_for_status: green + + - do: + indices.recovery: + index: [test_2] + human: true + + - match: { test_2.shards.0.type: "EXISTING_STORE" } + - match: { test_2.shards.0.stage: "DONE" } + - match: { test_2.shards.0.primary: true } + - match: { test_2.shards.0.start_time: /^2\d\d\d-.+/ } + - match: { test_2.shards.0.target.ip: /^\d+\.\d+\.\d+\.\d+$/ } + - gte: { test_2.shards.0.index.files.total: 0 } + - gte: { test_2.shards.0.index.files.reused: 0 } + - gte: { test_2.shards.0.index.files.recovered: 0 } + - match: { test_2.shards.0.index.files.percent: /^\d+\.\d\%$/ } + - gte: { test_2.shards.0.index.size.total_in_bytes: 0 } + - gte: { test_2.shards.0.index.size.reused_in_bytes: 0 } + - gte: { test_2.shards.0.index.size.recovered_in_bytes: 0 } + - match: { test_2.shards.0.index.size.percent: /^\d+\.\d\%$/ } + - gte: { test_2.shards.0.index.source_throttle_time_in_millis: 0 } + - gte: { test_2.shards.0.index.target_throttle_time_in_millis: 0 } + - gte: { test_2.shards.0.translog.recovered: 0 } + - gte: { test_2.shards.0.translog.total: -1 } + - gte: { test_2.shards.0.translog.total_on_start: -1 } + - gte: { test_2.shards.0.translog.total_time_in_millis: 0 } + - gte: { test_2.shards.0.verify_index.check_index_time_in_millis: 0 } + - gte: { test_2.shards.0.verify_index.total_time_in_millis: 0 } +--- "Indices recovery test index name not matching": - do: diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java index 8878713765ba0..78d2969c2fde4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.recovery; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.broadcast.BroadcastRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -47,7 +48,7 @@ public RecoveryRequest() { * @param indices Comma-separated list of indices about which to gather recovery information */ public RecoveryRequest(String... indices) { - super(indices); + super(indices, IndicesOptions.STRICT_EXPAND_OPEN_CLOSED); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index f7356bd242d06..0ff31f42b9295 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -112,11 +112,11 @@ protected ShardsIterator shards(ClusterState state, RecoveryRequest request, Str @Override protected ClusterBlockException checkGlobalBlock(ClusterState state, RecoveryRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); } @Override protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java index a04d2edc8dc63..6cf42e7ad3f14 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java @@ -40,6 +40,11 @@ protected BroadcastRequest(String[] indices) { this.indices = indices; } + protected BroadcastRequest(String[] indices, IndicesOptions indicesOptions) { + this.indices = indices; + this.indicesOptions = indicesOptions; + } + @Override public String[] indices() { return indices; diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index ea3e933a88314..b9163d5354934 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -54,6 +54,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -209,24 +210,34 @@ public void testGatewayRecoveryTestActiveOnly() throws Exception { } public void testReplicaRecovery() throws Exception { - logger.info("--> start node A"); - String nodeA = internalCluster().startNode(); + final String nodeA = internalCluster().startNode(); + createIndex(INDEX_NAME, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT) + .build()); + ensureGreen(INDEX_NAME); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (BackgroundIndexer indexer = new BackgroundIndexer(INDEX_NAME, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } - logger.info("--> create index on node: {}", nodeA); - createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT); + refresh(INDEX_NAME); + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs); - logger.info("--> start node B"); - String nodeB = internalCluster().startNode(); - ensureGreen(); + final IndexMetaData.State indexState = randomFrom(IndexMetaData.State.values()); + if (indexState == IndexMetaData.State.CLOSE) { + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + ensureGreen(INDEX_NAME); + } // force a shard recovery from nodeA to nodeB - logger.info("--> bump replica count"); - client().admin().indices().prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet(); - ensureGreen(); + final String nodeB = internalCluster().startNode(); + assertAcked(client().admin().indices().prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); - logger.info("--> request recoveries"); - RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); + final RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); // we should now have two total shards, one primary and one replica List recoveryStates = response.shardRecoveryStates().get(INDEX_NAME); @@ -238,14 +249,27 @@ public void testReplicaRecovery() throws Exception { assertThat(nodeBResponses.size(), equalTo(1)); // validate node A recovery - RecoveryState nodeARecoveryState = nodeAResponses.get(0); - assertRecoveryState(nodeARecoveryState, 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); + final RecoveryState nodeARecoveryState = nodeAResponses.get(0); + final RecoverySource expectedRecoverySource; + if (indexState == IndexMetaData.State.OPEN) { + expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE; + } else { + expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE; + } + assertRecoveryState(nodeARecoveryState, 0, expectedRecoverySource, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryState.getIndex()); // validate node B recovery - RecoveryState nodeBRecoveryState = nodeBResponses.get(0); + final RecoveryState nodeBRecoveryState = nodeBResponses.get(0); assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB); validateIndexRecoveryState(nodeBRecoveryState.getIndex()); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); + + if (indexState == IndexMetaData.State.CLOSE) { + assertAcked(client().admin().indices().prepareOpen(INDEX_NAME)); + } + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs); } @TestLogging( From 70eaee54581c758a508843986ec009e38bfd125e Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 21 Feb 2019 18:06:40 +0100 Subject: [PATCH 2/4] Apply feedback --- .../resources/rest-api-spec/test/cat.recovery/10_basic.yml | 4 ++-- .../rest-api-spec/test/indices.recovery/10_basic.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index f635932c34433..de0174bc801b8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -125,9 +125,9 @@ \d+ \s+ # bytes_recovered \d+\.\d+% \s+ # bytes_percent \d+ \s+ # bytes_total - -1 \s+ # translog_ops (always -1 for closed indices) + 0 \s+ # translog_ops (always 0 for closed indices) 0 \s+ # translog_ops_recovered (always 0 for closed indices) - -1\.0% # translog_ops_percent (always -1.0% for closed indices) + 100\.0% # translog_ops_percent (always 100.0% for closed indices) \n )+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml index c5df3b053583f..07fe657e77ff3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml @@ -84,8 +84,8 @@ - gte: { test_2.shards.0.index.source_throttle_time_in_millis: 0 } - gte: { test_2.shards.0.index.target_throttle_time_in_millis: 0 } - gte: { test_2.shards.0.translog.recovered: 0 } - - gte: { test_2.shards.0.translog.total: -1 } - - gte: { test_2.shards.0.translog.total_on_start: -1 } + - gte: { test_2.shards.0.translog.total: 0 } + - gte: { test_2.shards.0.translog.total_on_start: 0 } - gte: { test_2.shards.0.translog.total_time_in_millis: 0 } - gte: { test_2.shards.0.verify_index.check_index_time_in_millis: 0 } - gte: { test_2.shards.0.verify_index.total_time_in_millis: 0 } From a534a1d2d09420299b14db196ca1aeccdd0d5152 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 22 Feb 2019 12:53:49 +0100 Subject: [PATCH 3/4] Fix cat/recovery test --- .../rest-api-spec/test/cat.recovery/10_basic.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index de0174bc801b8..83194db1fa5a3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -83,11 +83,12 @@ reason: closed indices are replicated starting version 8.0.0 - do: - index: + indices.create: index: index2 - id: 1 - body: { foo: bar } - refresh: true + body: + settings: + index: + number_of_replicas: 0 - do: indices.close: @@ -97,7 +98,7 @@ - do: cluster.health: index: index2 - wait_for_status: yellow + wait_for_status: green - do: cat.recovery: From e4082dbb73c7e302aa2f547cd55556eb576c8904 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 22 Feb 2019 13:26:19 +0100 Subject: [PATCH 4/4] closedIndex --- .../elasticsearch/indices/recovery/IndexRecoveryIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index b9163d5354934..82d6c38becaec 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -225,8 +225,8 @@ public void testReplicaRecovery() throws Exception { refresh(INDEX_NAME); assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs); - final IndexMetaData.State indexState = randomFrom(IndexMetaData.State.values()); - if (indexState == IndexMetaData.State.CLOSE) { + final boolean closedIndex = randomBoolean(); + if (closedIndex) { assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); ensureGreen(INDEX_NAME); } @@ -251,7 +251,7 @@ public void testReplicaRecovery() throws Exception { // validate node A recovery final RecoveryState nodeARecoveryState = nodeAResponses.get(0); final RecoverySource expectedRecoverySource; - if (indexState == IndexMetaData.State.OPEN) { + if (closedIndex == false) { expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE; } else { expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE; @@ -266,7 +266,7 @@ public void testReplicaRecovery() throws Exception { internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); - if (indexState == IndexMetaData.State.CLOSE) { + if (closedIndex) { assertAcked(client().admin().indices().prepareOpen(INDEX_NAME)); } assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs);