From d3d2af969e65025788043c9c3aa43f1db9719d46 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 6 Aug 2020 11:17:02 +0100 Subject: [PATCH 1/8] Separate SearchableAction ITs into own class --- .../test/rest/ESRestTestCase.java | 2 +- .../xpack/TimeSeriesRestDriver.java | 23 +++ .../ilm/TimeSeriesLifecycleActionsIT.java | 188 +++++------------- .../actions/SearchableSnapshotActionIT.java | 139 +++++++++++++ 4 files changed, 214 insertions(+), 138 deletions(-) create mode 100644 x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 80ea7f439e66c..a772d33d685dd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1027,7 +1027,7 @@ protected static void assertOK(Response response) { * in an non green state * @param index index to test for **/ - protected static void ensureGreen(String index) throws IOException { + public static void ensureGreen(String index) throws IOException { ensureHealth(index, (request) -> { request.addParameter("wait_for_status", "green"); request.addParameter("wait_for_no_relocating_shards", "true"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 617524fa48368..3030c3337c147 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -42,6 +43,8 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.rest.ESRestTestCase.ensureGreen; /** * This class provides the operational REST functions needed to control an ILM time series lifecycle. @@ -204,4 +207,24 @@ public static Map getOnlyIndexSettings(RestClient client, String } } + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings) + throws IOException { + createIndexWithSettings(client, index, alias, settings, randomBoolean()); + } + + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings, + boolean useWriteIndex) throws IOException { + Request request = new Request("PUT", "/" + index); + + String writeIndexSnippet = ""; + if (useWriteIndex) { + writeIndexSnippet = "\"is_write_index\": true"; + } + request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build()) + + ", \"aliases\" : { \"" + alias + "\": { " + writeIndexSnippet + " } } }"); + client.performRequest(request); + // wait for the shards to initialize + ensureGreen(index); + } + } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 0adf7f87385e9..6438c39ad7928 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -8,7 +8,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Request; @@ -67,6 +66,7 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createFullPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explain; @@ -113,7 +113,7 @@ public void testFullPolicy() throws Exception { String originalIndex = index + "-000001"; String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -141,7 +141,7 @@ public void testFullPolicy() throws Exception { public void testMoveToAllocateStep() throws Exception { String originalIndex = index + "-000001"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias")); @@ -175,7 +175,7 @@ public void testMoveToRolloverStep() throws Exception { String originalIndex = index + "-000001"; String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -219,7 +219,7 @@ public void testMoveToRolloverStep() throws Exception { public void testRetryFailedDeleteAction() throws Exception { createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) @@ -239,7 +239,7 @@ public void testRetryFailedDeleteAction() throws Exception { public void testRetryFreezeDeleteAction() throws Exception { createNewSingletonPolicy(client(), policy, "cold", new FreezeAction()); - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) @@ -261,7 +261,7 @@ public void testRetryFailedShrinkAction() throws Exception { int divisor = randomFrom(2, 4); int expectedFinalShards = numShards / divisor; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numShards + randomIntBetween(1, numShards))); updatePolicy(index, policy); @@ -294,7 +294,7 @@ public void testRetryFailedShrinkAction() throws Exception { public void testRolloverAction() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -313,7 +313,7 @@ public void testRolloverAction() throws Exception { public void testRolloverActionWithIndexingComplete() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -352,7 +352,7 @@ public void testRolloverActionWithIndexingComplete() throws Exception { } public void testAllocateOnlyAllocation() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String allocateNodeName = "integTest-" + randomFrom(0, 1); AllocateAction allocateAction = new AllocateAction(null, null, null, singletonMap("_name", allocateNodeName)); @@ -369,7 +369,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { int numShards = randomFrom(1, 5); int numReplicas = randomFrom(0, 1); int finalNumReplicas = (numReplicas + 1) % 2; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)); AllocateAction allocateAction = new AllocateAction(finalNumReplicas, null, null, null); String endPhase = randomFrom("warm", "cold"); @@ -383,7 +383,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { } public void testWaitForSnapshot() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String slmPolicy = randomAlphaOfLengthBetween(4, 10); createNewSingletonPolicy(client(), policy, "delete", new WaitForSnapshotAction(slmPolicy)); @@ -413,7 +413,7 @@ public void testWaitForSnapshot() throws Exception { } public void testWaitForSnapshotSlmExecutedBefore() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String slmPolicy = randomAlphaOfLengthBetween(4, 10); createNewSingletonPolicy(client(), policy, "delete", new WaitForSnapshotAction(slmPolicy)); @@ -459,7 +459,7 @@ public void testWaitForSnapshotSlmExecutedBefore() throws Exception { } public void testDelete() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); updatePolicy(index, policy); @@ -467,7 +467,7 @@ public void testDelete() throws Exception { } public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction(), TimeValue.timeValueHours(1)); updatePolicy(index, policy); @@ -496,7 +496,7 @@ public void testDeleteDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "delete", new DeleteAction(), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); // index document so snapshot actually does something indexDocument(client(), index); @@ -516,7 +516,7 @@ public void testDeleteDuringSnapshot() throws Exception { } public void testReadOnly() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ReadOnlyAction()); updatePolicy(index, policy); @@ -529,7 +529,7 @@ public void testReadOnly() throws Exception { @SuppressWarnings("unchecked") public void forceMergeActionWithCodec(String codec) throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); for (int i = 0; i < randomIntBetween(2, 10); i++) { Request request = new Request("PUT", index + "/_doc/" + i); @@ -572,7 +572,7 @@ public void testShrinkAction() throws Exception { int divisor = randomFrom(2, 4); int expectedFinalShards = numShards / divisor; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards)); updatePolicy(index, policy); @@ -591,7 +591,7 @@ public void testShrinkAction() throws Exception { public void testShrinkSameShards() throws Exception { int numberOfShards = randomFrom(1, 2); String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numberOfShards)); updatePolicy(index, policy); @@ -625,7 +625,7 @@ public void testShrinkDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(1), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // required so the shrink doesn't wait on SetSingleNodeAllocateStep @@ -659,7 +659,7 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception int numShards = 2; int expectedFinalShards = 1; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); ensureGreen(index); @@ -711,7 +711,7 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception } public void testFreezeAction() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "cold", new FreezeAction()); updatePolicy(index, policy); @@ -741,7 +741,7 @@ public void testFreezeDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "cold", new FreezeAction(), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); // index document so snapshot actually does something indexDocument(client(), index); @@ -766,7 +766,7 @@ public void testFreezeDuringSnapshot() throws Exception { } public void testSetPriority() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100)); int priority = randomIntBetween(0, 99); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction(priority)); @@ -779,7 +779,7 @@ public void testSetPriority() throws Exception { } public void testSetNullPriority() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100)); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction((Integer) null)); updatePolicy(index, policy); @@ -904,7 +904,9 @@ public void testRemoveAndReaddPolicy() throws Exception { // Set up a policy with rollover createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); createIndexWithSettings( + client(), originalIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -940,7 +942,7 @@ public void testMoveToInjectedStep() throws Exception { String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(1), TimeValue.timeValueHours(12)); - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -976,7 +978,7 @@ public void testMoveToInjectedStep() throws Exception { public void testMoveToStepRereadsPolicy() throws Exception { createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, TimeValue.timeValueHours(1), null), TimeValue.ZERO); - createIndexWithSettings("test-1", Settings.builder() + createIndexWithSettings(client(), "test-1", alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1011,7 +1013,7 @@ public void testMoveToStepRereadsPolicy() throws Exception { } public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5,15))); @@ -1057,7 +1059,7 @@ public void testExplainFilters() throws Exception { assertOK(client().performRequest(request)); } - createIndexWithSettings(goodIndex, Settings.builder() + createIndexWithSettings(client(), goodIndex, alias, Settings.builder() .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1113,7 +1115,9 @@ public void testILMRolloverRetriesOnReadOnlyBlock() throws Exception { // create the index as readonly and associate the ILM policy to it createIndexWithSettings( + client(), firstIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1159,7 +1163,9 @@ public void testILMRolloverOnManuallyRolledIndex() throws Exception { client().performRequest(createIndexTemplate); createIndexWithSettings( + client(), originalIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), true @@ -1204,7 +1210,9 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except // create the rolled index so the rollover of the first index fails createIndexWithSettings( + client(), rolledIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), @@ -1212,7 +1220,9 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except ); createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1276,7 +1286,9 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1328,7 +1340,7 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( public void testWaitForActiveShardsStep() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), true); @@ -1375,7 +1387,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { "}"); client().performRequest(createIndexTemplate); - createIndexWithSettings(index + "-1", Settings.builder(), true); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), true); // Index a document index(client(), index + "-1", "1", "foo", "bar"); @@ -1401,7 +1413,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353") public void testHistoryIsWrittenWithFailure() throws Exception { - createIndexWithSettings(index + "-1", Settings.builder(), false); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); updatePolicy(index + "-1", policy); @@ -1420,7 +1432,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53718") public void testHistoryIsWrittenWithDeletion() throws Exception { // Index should be created and then deleted by ILM - createIndexWithSettings(index, Settings.builder(), false); + createIndexWithSettings(client(), index, alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); updatePolicy(index, policy); @@ -1444,7 +1456,9 @@ public void testRetryableInitializationStep() throws Exception { // Create the index with the origination parsing turn *off* so it doesn't prevent creation createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1495,7 +1509,7 @@ public void testRefreshablePhaseJson() throws Exception { "}"); client().performRequest(createIndexTemplate); - createIndexWithSettings(index + "-1", + createIndexWithSettings(client(), index + "-1", alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), true); @@ -1519,7 +1533,7 @@ public void testHaltAtEndOfPhase() throws Exception { createNewSingletonPolicy(client(), policy, "hot", new SetPriorityAction(100)); - createIndexWithSettings(index, + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -1551,88 +1565,6 @@ public void testHaltAtEndOfPhase() throws Exception { assertBusy(() -> assertFalse("expected " + index + " to be deleted by ILM", indexExists(index))); } - public void testSearchableSnapshotAction() throws Exception { - String snapshotRepo = randomAlphaOfLengthBetween(4, 10); - createSnapshotRepo(client(), snapshotRepo, randomBoolean()); - createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo)); - - createIndexWithSettings(index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); - - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; - assertTrue(waitUntil(() -> { - try { - return indexExists(restoredIndexName); - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - - assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, - TimeUnit.SECONDS); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") - public void testDeleteActionDeletesSearchableSnapshot() throws Exception { - String snapshotRepo = randomAlphaOfLengthBetween(4, 10); - createSnapshotRepo(client(), snapshotRepo, randomBoolean()); - - // create policy with cold and delete phases - Map coldActions = - Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); - Map phases = new HashMap<>(); - phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); - phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), singletonMap(DeleteAction.NAME, - new DeleteAction(true)))); - LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); - // PUT policy - XContentBuilder builder = jsonBuilder(); - lifecyclePolicy.toXContent(builder, null); - final StringEntity entity = new StringEntity( - "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); - Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); - createPolicyRequest.setEntity(entity); - assertOK(client().performRequest(createPolicyRequest)); - - createIndexWithSettings(index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); - - String[] snapshotName = new String[1]; - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; - assertTrue(waitUntil(() -> { - try { - Map explainIndex = explainIndex(client(), index); - if(explainIndex == null) { - // in case we missed the original index and it was deleted - explainIndex = explainIndex(client(), restoredIndexName); - } - snapshotName[0] = (String) explainIndex.get("snapshot_name"); - return snapshotName[0] != null; - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - assertBusy(() -> assertFalse(indexExists(restoredIndexName))); - - assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> { - try { - Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]); - Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); - return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception"); - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - } - @SuppressWarnings("unchecked") public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -1655,7 +1587,7 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { createPolicyRequest.setEntity(entity); assertOK(client().performRequest(createPolicyRequest)); - createIndexWithSettings(index, + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -1812,24 +1744,6 @@ private void createIndexWithSettingsNoAlias(String index, Settings.Builder setti ensureGreen(index); } - private void createIndexWithSettings(String index, Settings.Builder settings) throws IOException { - createIndexWithSettings(index, settings, randomBoolean()); - } - - private void createIndexWithSettings(String index, Settings.Builder settings, boolean useWriteIndex) throws IOException { - Request request = new Request("PUT", "/" + index); - - String writeIndexSnippet = ""; - if (useWriteIndex) { - writeIndexSnippet = "\"is_write_index\": true"; - } - request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build()) - + ", \"aliases\" : { \"" + alias + "\": { " + writeIndexSnippet + " } } }"); - client().performRequest(request); - // wait for the shards to initialize - ensureGreen(index); - } - private static void index(RestClient client, String index, String id, Object... fields) throws IOException { XContentBuilder document = jsonBuilder().startObject(); for (int i = 0; i < fields.length; i += 2) { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java new file mode 100644 index 0000000000000..61ffa72e23878 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ilm.actions; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.hamcrest.Matchers.is; + +public class SearchableSnapshotActionIT extends ESRestTestCase { + + private String index; + private String policy; + private String alias; + + @Before + public void refreshIndex() { + index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + policy = "policy-" + randomAlphaOfLength(5); + alias = "alias-" + randomAlphaOfLength(5); + } + + public void testSearchableSnapshotAction() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo)); + + createIndexWithSettings(client(), index, alias, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + randomBoolean()); + + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, + TimeUnit.SECONDS); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") + public void testDeleteActionDeletesSearchableSnapshot() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + + // create policy with cold and delete phases + Map coldActions = + Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); + Map phases = new HashMap<>(); + phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); + phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), singletonMap(DeleteAction.NAME, + new DeleteAction(true)))); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); + // PUT policy + XContentBuilder builder = jsonBuilder(); + lifecyclePolicy.toXContent(builder, null); + final StringEntity entity = new StringEntity( + "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); + Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); + createPolicyRequest.setEntity(entity); + assertOK(client().performRequest(createPolicyRequest)); + + createIndexWithSettings(client(), index, alias, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME, policy), + randomBoolean()); + + String[] snapshotName = new String[1]; + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; + assertTrue(waitUntil(() -> { + try { + Map explainIndex = explainIndex(client(), index); + if(explainIndex == null) { + // in case we missed the original index and it was deleted + explainIndex = explainIndex(client(), restoredIndexName); + } + snapshotName[0] = (String) explainIndex.get("snapshot_name"); + return snapshotName[0] != null; + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + assertBusy(() -> assertFalse(indexExists(restoredIndexName))); + + assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> { + try { + Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]); + Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); + return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception"); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + } + + +} From 120c53d532d82a1256aa697e10c86aa9f46c64ab Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 6 Aug 2020 12:58:50 +0100 Subject: [PATCH 2/8] ILM: add force-merge step to searchable snapshots action This adds a force-merge step to the searchable snapshot action, enabled by default, but parameterizable using the `force_merge-index" optional boolean. eg. ``` PUT _ilm/policy/my_policy { "policy": { "phases": { "cold": { "actions": { "searchable_snapshot" : { "snapshot_repository" : "backing_repo", "force_merge_index": true } } } } } } ``` --- .../actions/ilm-searchable-snapshot.asciidoc | 8 ++ .../core/ilm/SearchableSnapshotAction.java | 59 +++++++++-- .../xpack/core/ilm/SegmentCountStep.java | 3 +- .../ilm/SearchableSnapshotActionTests.java | 97 ++++++++++++------- .../xpack/TimeSeriesRestDriver.java | 13 +++ .../xpack/ilm/TimeSeriesDataStreamsIT.java | 4 +- .../ilm/TimeSeriesLifecycleActionsIT.java | 19 +--- .../actions/SearchableSnapshotActionIT.java | 48 ++++++++- 8 files changed, 188 insertions(+), 63 deletions(-) diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 20aa7ae2f1d6f..bfaf19b02780e 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -30,6 +30,14 @@ To keep the snapshot, set `delete_searchable_snapshot` to `false` in the delete Specifies where to store the snapshot. See <> for more information. +`force_merge_index`:: +(Optional, boolean) +Force merges the managed index to one segment. +Defaults to `true`. +If the managed index was already force merged using the +<> in the `cold` or `warm` phase +the `searchable snapshot` action force merge will be a no-op. + [[ilm-searchable-snapshot-ex]] ==== Examples [source,console] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 36716fc6941ed..730e854ba69b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; @@ -18,7 +19,7 @@ import org.elasticsearch.xpack.core.ilm.Step.StepKey; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -31,15 +32,17 @@ public class SearchableSnapshotAction implements LifecycleAction { public static final String NAME = "searchable_snapshot"; public static final ParseField SNAPSHOT_REPOSITORY = new ParseField("snapshot_repository"); + public static final ParseField FORCE_MERGE_INDEX = new ParseField("force_merge_index"); public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; public static final String RESTORED_INDEX_PREFIX = "restored-"; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, - a -> new SearchableSnapshotAction((String) a[0])); + a -> new SearchableSnapshotAction((String) a[0], a[1] == null || (boolean) a[1])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), SNAPSHOT_REPOSITORY); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), FORCE_MERGE_INDEX); } public static SearchableSnapshotAction parse(XContentParser parser) { @@ -47,22 +50,34 @@ public static SearchableSnapshotAction parse(XContentParser parser) { } private final String snapshotRepository; + private final boolean forceMergeIndex; - public SearchableSnapshotAction(String snapshotRepository) { + public SearchableSnapshotAction(String snapshotRepository, boolean forceMergeIndex) { if (Strings.hasText(snapshotRepository) == false) { throw new IllegalArgumentException("the snapshot repository must be specified"); } this.snapshotRepository = snapshotRepository; + this.forceMergeIndex = forceMergeIndex; + } + + public SearchableSnapshotAction(String snapshotRepository) { + this(snapshotRepository, true); } public SearchableSnapshotAction(StreamInput in) throws IOException { - this(in.readString()); + this(in.readString(), in.getVersion().onOrAfter(Version.CURRENT) ? in.readBoolean() : true); + } + + boolean isForceMergeIndex() { + return forceMergeIndex; } @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNoWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); + StepKey forceMergeStepKey = new StepKey(phase, NAME, ForceMergeStep.NAME); + StepKey waitForSegmentCountKey = new StepKey(phase, NAME, SegmentCountStep.NAME); StepKey generateSnapshotNameKey = new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME); StepKey cleanSnapshotKey = new StepKey(phase, NAME, CleanupSnapshotStep.NAME); StepKey createSnapshotKey = new StepKey(phase, NAME, CreateSnapshotStep.NAME); @@ -77,8 +92,14 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex, waitForNoFollowerStepKey); - WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, generateSnapshotNameKey, - client); + WaitForNoFollowersStep waitForNoFollowersStep; + if (forceMergeIndex) { + waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, forceMergeStepKey, client); + } else { + waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, generateSnapshotNameKey, client); + } + ForceMergeStep forceMergeStep = new ForceMergeStep(forceMergeStepKey, waitForSegmentCountKey, client, 1); + SegmentCountStep segmentCountStep = new SegmentCountStep(waitForSegmentCountKey, generateSnapshotNameKey, client, 1); GenerateSnapshotNameStep generateSnapshotNameStep = new GenerateSnapshotNameStep(generateSnapshotNameKey, cleanSnapshotKey, snapshotRepository); CleanupSnapshotStep cleanupSnapshotStep = new CleanupSnapshotStep(cleanSnapshotKey, createSnapshotKey, client); @@ -108,9 +129,25 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { SwapAliasesAndDeleteSourceIndexStep swapAliasesAndDeleteSourceIndexStep = new SwapAliasesAndDeleteSourceIndexStep(swapAliasesKey, null, client, RESTORED_INDEX_PREFIX); - return Arrays.asList(checkNoWriteIndexStep, waitForNoFollowersStep, generateSnapshotNameStep, cleanupSnapshotStep, - createSnapshotBranchingStep, mountSnapshotStep, waitForGreenIndexHealthStep, copyMetadataStep, copySettingsStep, - isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep, swapAliasesAndDeleteSourceIndexStep); + List steps = new ArrayList<>(); + steps.add(checkNoWriteIndexStep); + steps.add(waitForNoFollowersStep); + if (forceMergeIndex) { + steps.add(forceMergeStep); + steps.add(segmentCountStep); + } + steps.add(generateSnapshotNameStep); + steps.add(cleanupSnapshotStep); + steps.add(createSnapshotBranchingStep); + steps.add(mountSnapshotStep); + steps.add(waitForGreenIndexHealthStep); + steps.add(copyMetadataStep); + steps.add(copySettingsStep); + steps.add(isDataStreamBranchingStep); + steps.add(replaceDataStreamBackingIndex); + steps.add(deleteSourceIndexStep); + steps.add(swapAliasesAndDeleteSourceIndexStep); + return steps; } @Override @@ -126,12 +163,16 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(snapshotRepository); + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeBoolean(forceMergeIndex); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(SNAPSHOT_REPOSITORY.getPreferredName(), snapshotRepository); + builder.field(FORCE_MERGE_INDEX.getPreferredName(), forceMergeIndex); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java index 9978d4cec21ba..81c66238c2e7f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java @@ -57,8 +57,7 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener, if (idxSegments == null || (response.getShardFailures() != null && response.getShardFailures().length > 0)) { final DefaultShardOperationFailedException[] failures = response.getShardFailures(); logger.info("[{}] retrieval of segment counts after force merge did not succeed, " + - "there were {} shard failures. " + - "failures: {}", + "there were {} shard failures. failures: {}", index.getName(), response.getFailedShards(), failures == null ? "n/a" : Strings.collectionToDelimitedString(Arrays.stream(failures) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index 375f5d07ae297..48757fc483064 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -20,42 +20,73 @@ public class SearchableSnapshotActionTests extends AbstractActionTestCase steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), is(13)); - - assertThat(steps.get(0).getKey(), is(expectedFirstStep)); - assertThat(steps.get(1).getKey(), is(expectedSecondStep)); - assertThat(steps.get(2).getKey(), is(expectedThirdStep)); - assertThat(steps.get(3).getKey(), is(expectedFourthStep)); - assertThat(steps.get(4).getKey(), is(expectedFifthStep)); - assertThat(steps.get(5).getKey(), is(expectedSixthStep)); - assertThat(steps.get(6).getKey(), is(expectedSeventhStep)); - assertThat(steps.get(7).getKey(), is(expectedEighthStep)); - assertThat(steps.get(8).getKey(), is(expectedNinthStep)); - assertThat(steps.get(9).getKey(), is(expectedTenthStep)); - assertThat(steps.get(10).getKey(), is(expectedElevenStep)); - assertThat(steps.get(11).getKey(), is(expectedTwelveStep)); - assertThat(steps.get(12).getKey(), is(expectedThirteenStep)); - - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedFourthStep)); + assertThat(steps.size(), is(action.isForceMergeIndex() ? 15 : 13)); + + List expectedSteps = action.isForceMergeIndex() ? expectedStepKeysWithForceMerge(phase) : + expectedStepKeysNoForceMerge(phase); + + assertThat(steps.get(0).getKey(), is(expectedSteps.get(0))); + assertThat(steps.get(1).getKey(), is(expectedSteps.get(1))); + assertThat(steps.get(2).getKey(), is(expectedSteps.get(2))); + assertThat(steps.get(3).getKey(), is(expectedSteps.get(3))); + assertThat(steps.get(4).getKey(), is(expectedSteps.get(4))); + assertThat(steps.get(5).getKey(), is(expectedSteps.get(5))); + assertThat(steps.get(6).getKey(), is(expectedSteps.get(6))); + assertThat(steps.get(7).getKey(), is(expectedSteps.get(7))); + assertThat(steps.get(8).getKey(), is(expectedSteps.get(8))); + assertThat(steps.get(9).getKey(), is(expectedSteps.get(9))); + assertThat(steps.get(10).getKey(), is(expectedSteps.get(10))); + assertThat(steps.get(11).getKey(), is(expectedSteps.get(11))); + assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); + + if (action.isForceMergeIndex()) { + assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(6); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(5))); + } else { + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(3))); + } + } + + private List expectedStepKeysWithForceMerge(String phase) { + return List.of( + new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), + new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), + new StepKey(phase, NAME, ForceMergeStep.NAME), + new StepKey(phase, NAME, SegmentCountStep.NAME), + new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), + new StepKey(phase, NAME, CleanupSnapshotStep.NAME), + new StepKey(phase, NAME, CreateSnapshotStep.NAME), + new StepKey(phase, NAME, MountSnapshotStep.NAME), + new StepKey(phase, NAME, WaitForIndexColorStep.NAME), + new StepKey(phase, NAME, CopyExecutionStateStep.NAME), + new StepKey(phase, NAME, CopySettingsStep.NAME), + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), + new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), + new StepKey(phase, NAME, DeleteStep.NAME), + new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME)); + } + + private List expectedStepKeysNoForceMerge(String phase) { + return List.of( + new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), + new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), + new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), + new StepKey(phase, NAME, CleanupSnapshotStep.NAME), + new StepKey(phase, NAME, CreateSnapshotStep.NAME), + new StepKey(phase, NAME, MountSnapshotStep.NAME), + new StepKey(phase, NAME, WaitForIndexColorStep.NAME), + new StepKey(phase, NAME, CopyExecutionStateStep.NAME), + new StepKey(phase, NAME, CopySettingsStep.NAME), + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), + new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), + new StepKey(phase, NAME, DeleteStep.NAME), + new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME)); } @Override @@ -79,6 +110,6 @@ protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction insta } static SearchableSnapshotAction randomInstance() { - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10)); + return new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), randomBoolean()); } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 3030c3337c147..4a57b9943b61d 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -37,6 +37,7 @@ import java.io.InputStream; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -227,4 +228,16 @@ public static void createIndexWithSettings(RestClient client, String index, Stri ensureGreen(index); } + @SuppressWarnings("unchecked") + public static Integer getNumberOfSegments(RestClient client, String index) throws IOException { + Response response = client.performRequest(new Request("GET", index + "/_segments")); + XContentType entityContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + Map responseEntity = XContentHelper.convertToMap(entityContentType.xContent(), + response.getEntity().getContent(), false); + responseEntity = (Map) responseEntity.get("indices"); + responseEntity = (Map) responseEntity.get(index); + responseEntity = (Map) responseEntity.get("shards"); + List> shards = (List>) responseEntity.get("0"); + return (Integer) shards.get(0).get("num_search_segments"); + } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index e5cd67ae528b8..b783dfd1f4598 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -223,10 +223,10 @@ public void testGetDataStreamReturnsILMPolicy() throws Exception { } private static Template getTemplate(String policyName) throws IOException { - return new Template(getLifcycleSettings(policyName), null, null); + return new Template(getLifecycleSettings(policyName), null, null); } - private static Settings getLifcycleSettings(String policyName) { + private static Settings getLifecycleSettings(String policyName) { return Settings.builder() .put(LifecycleSettings.LIFECYCLE_NAME, policyName) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 6438c39ad7928..69afbc53e2e82 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -61,7 +61,6 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -71,6 +70,7 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explain; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; @@ -527,7 +527,6 @@ public void testReadOnly() throws Exception { }); } - @SuppressWarnings("unchecked") public void forceMergeActionWithCodec(String codec) throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); @@ -538,26 +537,14 @@ public void forceMergeActionWithCodec(String codec) throws Exception { client().performRequest(request); } - Supplier numSegments = () -> { - try { - Map segmentResponse = getAsMap(index + "/_segments"); - segmentResponse = (Map) segmentResponse.get("indices"); - segmentResponse = (Map) segmentResponse.get(index); - segmentResponse = (Map) segmentResponse.get("shards"); - List> shards = (List>) segmentResponse.get("0"); - return (Integer) shards.get(0).get("num_search_segments"); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - assertThat(numSegments.get(), greaterThan(1)); + assertThat(getNumberOfSegments(client(), index), greaterThan(1)); createNewSingletonPolicy(client(), policy, "warm", new ForceMergeAction(1, codec)); updatePolicy(index, policy); assertBusy(() -> { assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("warm").getKey())); Map settings = getOnlyIndexSettings(client(), index); - assertThat(numSegments.get(), equalTo(1)); + assertThat(getNumberOfSegments(client(), index), equalTo(1)); assertThat(settings.get(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true")); }); expectThrows(ResponseException.class, () -> indexDocument(client(), index)); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 61ffa72e23878..0f317ed405926 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -11,7 +11,9 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -34,10 +36,15 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; public class SearchableSnapshotActionIT extends ESRestTestCase { @@ -78,6 +85,46 @@ public void testSearchableSnapshotAction() throws Exception { TimeUnit.SECONDS); } + public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true)); + + createComposableTemplate(client(), "template-name", index, new Template(null, null, null)); + + String dataStreamName = index; + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStreamName, true); + } + + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1L); + assertThat(getNumberOfSegments(client(), backingIndexName), greaterThan(1)); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStreamName); + + updateIndexSettings(dataStreamName, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)); + assertTrue(waitUntil(() -> { + try { + return getNumberOfSegments(client(), backingIndexName) == 1; + } catch (IOException e) { + return false; + } + }, 60, TimeUnit.SECONDS)); + + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, + TimeUnit.SECONDS); + } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") public void testDeleteActionDeletesSearchableSnapshot() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -135,5 +182,4 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { }, 30, TimeUnit.SECONDS)); } - } From 9a56ea1a04b3536938fb86b11190eda970185fb2 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 6 Aug 2020 13:30:10 +0100 Subject: [PATCH 3/8] SearchableSnapshotActionIT use data stream instead of index/alias --- .../actions/SearchableSnapshotActionIT.java | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 0f317ed405926..a8a9ebae2ab96 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -37,7 +36,6 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; -import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; @@ -49,30 +47,30 @@ public class SearchableSnapshotActionIT extends ESRestTestCase { - private String index; private String policy; - private String alias; + private String dataStream; @Before public void refreshIndex() { - index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + dataStream = "logs-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); policy = "policy-" + randomAlphaOfLength(5); - alias = "alias-" + randomAlphaOfLength(5); } public void testSearchableSnapshotAction() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); createSnapshotRepo(client(), snapshotRepo, randomBoolean()); - createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo)); + createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true)); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)); - createIndexWithSettings(client(), index, alias, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); + indexDocument(client(), dataStream, true); - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { return indexExists(restoredIndexName); @@ -90,20 +88,19 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio createSnapshotRepo(client(), snapshotRepo, randomBoolean()); createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true)); - createComposableTemplate(client(), "template-name", index, new Template(null, null, null)); + createComposableTemplate(client(), "template-name", dataStream, new Template(null, null, null)); - String dataStreamName = index; for (int i = 0; i < randomIntBetween(5, 10); i++) { - indexDocument(client(), dataStreamName, true); + indexDocument(client(), dataStream, true); } - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1L); + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); assertThat(getNumberOfSegments(client(), backingIndexName), greaterThan(1)); // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index - rolloverMaxOneDocCondition(client(), dataStreamName); + rolloverMaxOneDocCondition(client(), dataStream); - updateIndexSettings(dataStreamName, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)); + updateIndexSettings(dataStream, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)); assertTrue(waitUntil(() -> { try { return getNumberOfSegments(client(), backingIndexName) == 1; @@ -147,19 +144,21 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { createPolicyRequest.setEntity(entity); assertOK(client().performRequest(createPolicyRequest)); - createIndexWithSettings(client(), index, alias, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)); + + indexDocument(client(), dataStream, true); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); String[] snapshotName = new String[1]; - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { - Map explainIndex = explainIndex(client(), index); - if(explainIndex == null) { + Map explainIndex = explainIndex(client(), backingIndexName); + if (explainIndex == null) { // in case we missed the original index and it was deleted explainIndex = explainIndex(client(), restoredIndexName); } From b906a739ea9e5399b277d4cd49343032562ef447 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 6 Aug 2020 13:39:22 +0100 Subject: [PATCH 4/8] Update doc --- docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index bfaf19b02780e..1f500da520406 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -36,7 +36,7 @@ Force merges the managed index to one segment. Defaults to `true`. If the managed index was already force merged using the <> in the `cold` or `warm` phase -the `searchable snapshot` action force merge will be a no-op. +the `searchable snapshot` action force merge step will be a no-op. [[ilm-searchable-snapshot-ex]] ==== Examples From 4c24b4947e6ff27352864f19dc486c77eb1bfb16 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 6 Aug 2020 13:39:55 +0100 Subject: [PATCH 5/8] Use Version.V_8_0_0 --- .../xpack/core/ilm/SearchableSnapshotAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 730e854ba69b1..314bf4435bb52 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -65,7 +65,7 @@ public SearchableSnapshotAction(String snapshotRepository) { } public SearchableSnapshotAction(StreamInput in) throws IOException { - this(in.readString(), in.getVersion().onOrAfter(Version.CURRENT) ? in.readBoolean() : true); + this(in.readString(), in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : true); } boolean isForceMergeIndex() { @@ -163,7 +163,7 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(snapshotRepository); - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeBoolean(forceMergeIndex); } } From 6f856761300d99457c6d01440315c9690758f3df Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 10 Aug 2020 09:36:53 +0100 Subject: [PATCH 6/8] Make step final Co-authored-by: Lee Hinman --- .../elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 314bf4435bb52..43093eb269684 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -92,7 +92,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex, waitForNoFollowerStepKey); - WaitForNoFollowersStep waitForNoFollowersStep; + final WaitForNoFollowersStep waitForNoFollowersStep; if (forceMergeIndex) { waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, forceMergeStepKey, client); } else { From 0c65f2af32923452578d305558a2037c89d94dd2 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 10 Aug 2020 09:58:28 +0100 Subject: [PATCH 7/8] Update documentation to mention forcemerge is best effort --- docs/reference/ilm/actions/ilm-forcemerge.asciidoc | 4 ++++ .../ilm/actions/ilm-searchable-snapshot.asciidoc | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc index 86f8d579a1789..98c43d7ab6762 100644 --- a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc +++ b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc @@ -12,6 +12,10 @@ This action makes the index <>. To use the `forcemerge` action in the `hot` phase, the `rollover` action *must* be present. If no rollover action is configured, {ilm-init} will reject the policy. +[NOTE] +The `forcemerge` action is best effort. It might happen that some of the +shards are relocating, in which case they will not be merged. + [[ilm-forcemerge-options]] ==== Options diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 1f500da520406..ea5869696d6c7 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -35,9 +35,15 @@ See <> for more information. Force merges the managed index to one segment. Defaults to `true`. If the managed index was already force merged using the -<> in the `cold` or `warm` phase +<> in a previous action the `searchable snapshot` action force merge step will be a no-op. +[NOTE] +The `forcemerge` action is best effort. It might happen that some of +the shards are relocating, in which case they will not be merged. +The `searchable-snapshot` action will continue executing even if not all shards +are force merged. + [[ilm-searchable-snapshot-ex]] ==== Examples [source,console] From edfc0d5759e4aa2da2250aebccd25b0661332f2f Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 10 Aug 2020 09:59:50 +0100 Subject: [PATCH 8/8] Disable bwc tests --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 44f3d4559ec0a..b22c447df5d23 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/60819" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")