From 7744c7e00a07a09719c16cc366dfa41421231e7e Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 9 Jan 2020 07:37:05 +0100 Subject: [PATCH 1/6] ILM action to wait for SLM policy execution (#50454) This change add new ILM action to wait for SLM policy execution to ensure that index has snapshot before deletion. Closes #45067 --- .../reference/ilm/policy-definitions.asciidoc | 35 ++++ .../xpack/core/XPackClientPlugin.java | 2 + .../core/ilm/TimeseriesLifecycleType.java | 2 +- .../xpack/core/ilm/WaitForSnapshotAction.java | 96 +++++++++++ .../xpack/core/ilm/WaitForSnapshotStep.java | 101 ++++++++++++ .../ilm/LifecyclePolicyMetadataTests.java | 3 + .../xpack/core/ilm/LifecyclePolicyTests.java | 7 + .../ilm/TimeseriesLifecycleTypeTests.java | 3 + .../core/ilm/WaitForSnapshotActionTests.java | 56 +++++++ .../core/ilm/WaitForSnapshotStepTests.java | 149 ++++++++++++++++++ .../ilm/action/PutLifecycleRequestTests.java | 4 + .../ilm/TimeSeriesLifecycleActionsIT.java | 76 +++++++++ .../xpack/ilm/IndexLifecycle.java | 4 +- .../ilm/IndexLifecycleMetadataTests.java | 4 + 14 files changed, 540 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc index 67fcdba9b76ef..b8c4121278e16 100644 --- a/docs/reference/ilm/policy-definitions.asciidoc +++ b/docs/reference/ilm/policy-definitions.asciidoc @@ -113,6 +113,7 @@ policy definition. - <> - <> * Delete + - <> - <> [[ilm-allocate-action]] @@ -224,6 +225,40 @@ PUT _ilm/policy/my_policy } -------------------------------------------------- +[[ilm-wait-for-snapshot-action]] +==== Wait For Snapshot + +Phases allowed: delete. + +The Wait For Snapshot Action waits for defined SLM policy to be executed to ensure that snapshot of index exists before +deletion. + +[[ilm-wait-for-snapshot-options]] +.Wait For Snapshot +[options="header"] +|====== +| Name | Required | Default | Description +| `policy` | yes | - | SLM policy name that this action should wait for +|====== + +[source,console] +-------------------------------------------------- +PUT _ilm/policy/my_policy +{ + "policy": { + "phases": { + "delete": { + "actions": { + "wait_for_snapshot" : { + "policy": "slm-policy-name" + } + } + } + } + } +} +-------------------------------------------------- + [[ilm-delete-action]] ==== Delete diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 843e3e611df25..8c4b5af8c8c70 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -65,6 +65,7 @@ import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -588,6 +589,7 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(LifecycleAction.class, FreezeAction.NAME, FreezeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new), // Data Frame new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index b2948df6373b4..ac5b3f51287b5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -37,7 +37,7 @@ public class TimeseriesLifecycleType implements LifecycleType { AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, FreezeAction.NAME); - static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(DeleteAction.NAME); + static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(WaitForSnapshotAction.NAME, DeleteAction.NAME); static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); static final Set VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java new file mode 100644 index 0000000000000..566b54b470c6d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A {@link LifecycleAction} which waits for snapshot to be taken (by configured SLM policy). + */ +public class WaitForSnapshotAction implements LifecycleAction { + + public static final String NAME = "wait_for_snapshot"; + public static final ParseField POLICY_FIELD = new ParseField("policy"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + a -> new WaitForSnapshotAction((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_FIELD); + } + + private final String policy; + + public static WaitForSnapshotAction parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public WaitForSnapshotAction(String policy) { + if (Strings.hasText(policy) == false) { + throw new IllegalArgumentException("policy name must be specified"); + } + this.policy = policy; + } + + public WaitForSnapshotAction(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey waitForSnapshotKey = new StepKey(phase, NAME, WaitForSnapshotStep.NAME); + return Collections.singletonList(new WaitForSnapshotStep(waitForSnapshotKey, nextStepKey, policy)); + } + + @Override + public boolean isSafeAction() { + return true; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(policy); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(POLICY_FIELD.getPreferredName(), policy); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WaitForSnapshotAction that = (WaitForSnapshotAction) o; + return policy.equals(that.policy); + } + + @Override + public int hashCode() { + return Objects.hash(policy); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java new file mode 100644 index 0000000000000..2880de8ea3de5 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.index.Index; +import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; + +import java.util.Date; +import java.util.Locale; +import java.util.Objects; + +/*** + * A step that waits for snapshot to be taken by SLM to ensure we have backup before we delete the index. + * It will signal error if it can't get data needed to do the check (phase time from ILM and SLM metadata) + * and will only return success if execution of SLM policy took place after index entered deleted phase. + */ +public class WaitForSnapshotStep extends ClusterStateWaitStep { + + static final String NAME = "wait-for-snapshot"; + + private static final String MESSAGE_FIELD = "message"; + private static final String POLICY_NOT_EXECUTED_MESSAGE = "waiting for policy '%s' to be executed since %s"; + private static final String POLICY_NOT_FOUND_MESSAGE = "configured policy '%s' not found"; + private static final String NO_INDEX_METADATA_MESSAGE = "no index metadata found for index '%s'"; + private static final String NO_PHASE_TIME_MESSAGE = "no information about ILM phase start in index metadata for index '%s'"; + + private final String policy; + + WaitForSnapshotStep(StepKey key, StepKey nextStepKey, String policy) { + super(key, nextStepKey); + this.policy = policy; + } + + @Override + public Result isConditionMet(Index index, ClusterState clusterState) { + IndexMetaData indexMetaData = clusterState.metaData().index(index); + if (indexMetaData == null) { + throw error(NO_INDEX_METADATA_MESSAGE, index.getName()); + } + + Long phaseTime = LifecycleExecutionState.fromIndexMetadata(indexMetaData).getPhaseTime(); + + if (phaseTime == null) { + throw error(NO_PHASE_TIME_MESSAGE, index.getName()); + } + + SnapshotLifecycleMetadata snapMeta = clusterState.metaData().custom(SnapshotLifecycleMetadata.TYPE); + if (snapMeta == null || snapMeta.getSnapshotConfigurations().containsKey(policy) == false) { + throw error(POLICY_NOT_FOUND_MESSAGE, policy); + } + SnapshotLifecyclePolicyMetadata snapPolicyMeta = snapMeta.getSnapshotConfigurations().get(policy); + if (snapPolicyMeta.getLastSuccess() == null || snapPolicyMeta.getLastSuccess().getTimestamp() < phaseTime) { + return new Result(false, notExecutedMessage(phaseTime)); + } + + return new Result(true, null); + } + + public String getPolicy() { + return policy; + } + + @Override + public boolean isRetryable() { + return true; + } + + private ToXContentObject notExecutedMessage(long time) { + return (builder, params) -> { + builder.startObject(); + builder.field(MESSAGE_FIELD, String.format(Locale.ROOT, POLICY_NOT_EXECUTED_MESSAGE, policy, new Date(time))); + builder.endObject(); + return builder; + }; + } + + private IllegalStateException error(String message, Object... args) { + return new IllegalStateException(String.format(Locale.ROOT, message, args)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + WaitForSnapshotStep that = (WaitForSnapshotStep) o; + return policy.equals(that.policy); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), policy); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java index b21b59049a3f4..9b50616a128dd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java @@ -39,6 +39,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), @@ -57,6 +58,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE), (p) -> TimeseriesLifecycleType.INSTANCE), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(WaitForSnapshotAction.NAME), WaitForSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 5470a6d31a586..37b268c499dbb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -48,6 +48,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), @@ -66,6 +67,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE), (p) -> TimeseriesLifecycleType.INSTANCE), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(WaitForSnapshotAction.NAME), WaitForSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), @@ -110,6 +113,8 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Null return AllocateActionTests.randomInstance(); case DeleteAction.NAME: return new DeleteAction(); + case WaitForSnapshotAction.NAME: + return WaitForSnapshotActionTests.randomInstance(); case ForceMergeAction.NAME: return ForceMergeActionTests.randomInstance(); case ReadOnlyAction.NAME: @@ -160,6 +165,8 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l switch (action) { case AllocateAction.NAME: return AllocateActionTests.randomInstance(); + case WaitForSnapshotAction.NAME: + return WaitForSnapshotActionTests.randomInstance(); case DeleteAction.NAME: return new DeleteAction(); case ForceMergeAction.NAME: diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index a76a22fcc7821..3186a545f23a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -34,6 +34,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { private static final AllocateAction TEST_ALLOCATE_ACTION = new AllocateAction(2, Collections.singletonMap("node", "node1"),null, null); private static final DeleteAction TEST_DELETE_ACTION = new DeleteAction(); + private static final WaitForSnapshotAction TEST_WAIT_FOR_SNAPSHOT_ACTION = new WaitForSnapshotAction("policy"); private static final ForceMergeAction TEST_FORCE_MERGE_ACTION = new ForceMergeAction(1); private static final RolloverAction TEST_ROLLOVER_ACTION = new RolloverAction(new ByteSizeValue(1), null, null); private static final ShrinkAction TEST_SHRINK_ACTION = new ShrinkAction(1); @@ -556,6 +557,8 @@ private LifecycleAction getTestAction(String actionName) { switch (actionName) { case AllocateAction.NAME: return TEST_ALLOCATE_ACTION; + case WaitForSnapshotAction.NAME: + return TEST_WAIT_FOR_SNAPSHOT_ACTION; case DeleteAction.NAME: return TEST_DELETE_ACTION; case ForceMergeAction.NAME: diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java new file mode 100644 index 0000000000000..a35ac6b422dd6 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +public class WaitForSnapshotActionTests extends AbstractActionTestCase { + + @Override + public void testToSteps() { + WaitForSnapshotAction action = createTestInstance(); + Step.StepKey nextStep = new Step.StepKey("", "", ""); + List steps = action.toSteps(null, "delete", nextStep); + assertEquals(1, steps.size()); + Step step = steps.get(0); + assertTrue(step instanceof WaitForSnapshotStep); + assertEquals(nextStep, step.getNextStepKey()); + + Step.StepKey key = step.getKey(); + assertEquals("delete", key.getPhase()); + assertEquals(WaitForSnapshotAction.NAME, key.getAction()); + assertEquals(WaitForSnapshotStep.NAME, key.getName()); + } + + @Override + protected WaitForSnapshotAction doParseInstance(XContentParser parser) throws IOException { + return WaitForSnapshotAction.parse(parser); + } + + @Override + protected WaitForSnapshotAction createTestInstance() { + return randomInstance(); + } + + @Override + protected Writeable.Reader instanceReader() { + return WaitForSnapshotAction::new; + } + + @Override + protected WaitForSnapshotAction mutateInstance(WaitForSnapshotAction instance) throws IOException { + return randomInstance(); + } + + static WaitForSnapshotAction randomInstance() { + return new WaitForSnapshotAction(randomAlphaOfLengthBetween(5, 10)); + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java new file mode 100644 index 0000000000000..6c1822235ec97 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.xpack.core.slm.SnapshotInvocationRecord; +import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; + +import java.io.IOException; +import java.util.Map; + +public class WaitForSnapshotStepTests extends AbstractStepTestCase { + + @Override + protected WaitForSnapshotStep createRandomInstance() { + return new WaitForSnapshotStep(randomStepKey(), randomStepKey(), randomAlphaOfLengthBetween(1, 10)); + } + + @Override + protected WaitForSnapshotStep mutateInstance(WaitForSnapshotStep instance) { + Step.StepKey key = instance.getKey(); + Step.StepKey nextKey = instance.getNextStepKey(); + String policy = instance.getPolicy(); + + switch (between(0, 2)) { + case 0: + key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 2: + policy = randomAlphaOfLengthBetween(1, 10); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new WaitForSnapshotStep(key, nextKey, policy); + } + + @Override + protected WaitForSnapshotStep copyInstance(WaitForSnapshotStep instance) { + return new WaitForSnapshotStep(instance.getKey(), instance.getNextStepKey(), instance.getPolicy()); + } + + public void testNoSlmPolicies() { + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(randomLong()))) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + ImmutableOpenMap.Builder indices = + ImmutableOpenMap.builder().fPut(indexMetaData.getIndex().getName(), indexMetaData); + MetaData.Builder meta = MetaData.builder().indices(indices.build()); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(meta).build(); + WaitForSnapshotStep instance = createRandomInstance(); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> instance.isConditionMet(indexMetaData.getIndex(), + clusterState)); + assertTrue(e.getMessage().contains(instance.getPolicy())); + } + + public void testSlmPolicyNotExecuted() throws IOException { + WaitForSnapshotStep instance = createRandomInstance(); + SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() + .setModifiedDate(randomLong()) + .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) + .build(); + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + OperationMode.RUNNING, null); + + + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(randomLong()))) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + ImmutableOpenMap.Builder indices = + ImmutableOpenMap.builder().fPut(indexMetaData.getIndex().getName(), indexMetaData); + MetaData.Builder meta = MetaData.builder().indices(indices.build()).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetaData); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(meta).build(); + ClusterStateWaitStep.Result result = instance.isConditionMet(indexMetaData.getIndex(), clusterState); + assertFalse(result.isComplete()); + assertTrue(getMessage(result).contains("to be executed")); + } + + public void testSlmPolicyExecutedBeforeStep() throws IOException { + long phaseTime = randomLong(); + + WaitForSnapshotStep instance = createRandomInstance(); + SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() + .setModifiedDate(randomLong()) + .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) + .setLastSuccess(new SnapshotInvocationRecord("", phaseTime - 10, "")) + .build(); + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + OperationMode.RUNNING, null); + + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(phaseTime))) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + ImmutableOpenMap.Builder indices = + ImmutableOpenMap.builder().fPut(indexMetaData.getIndex().getName(), indexMetaData); + MetaData.Builder meta = MetaData.builder().indices(indices.build()).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetaData); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(meta).build(); + ClusterStateWaitStep.Result result = instance.isConditionMet(indexMetaData.getIndex(), clusterState); + assertFalse(result.isComplete()); + assertTrue(getMessage(result).contains("to be executed")); + } + + public void testSlmPolicyExecutedAfterStep() throws IOException { + long phaseTime = randomLong(); + + WaitForSnapshotStep instance = createRandomInstance(); + SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() + .setModifiedDate(randomLong()) + .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) + .setLastSuccess(new SnapshotInvocationRecord("", phaseTime + 10, "")) + .build(); + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + OperationMode.RUNNING, null); + + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(phaseTime))) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + ImmutableOpenMap.Builder indices = + ImmutableOpenMap.builder().fPut(indexMetaData.getIndex().getName(), indexMetaData); + MetaData.Builder meta = MetaData.builder().indices(indices.build()).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetaData); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(meta).build(); + ClusterStateWaitStep.Result result = instance.isConditionMet(indexMetaData.getIndex(), clusterState); + assertTrue(result.isComplete()); + assertNull(result.getInfomationContext()); + } + + private String getMessage(ClusterStateWaitStep.Result result) throws IOException { + return Strings.toString(result.getInfomationContext()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java index 137a81ba56f2d..f0fd2f7dd5d76 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction.Request; import org.junit.Before; @@ -64,6 +65,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), @@ -82,6 +84,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE), (p) -> TimeseriesLifecycleType.INSTANCE), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(WaitForSnapshotAction.NAME), WaitForSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index b7fc5bdcb69e8..6d11c151e4e7e 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.ShrinkStep; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.Step; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.ilm.TerminalPolicyStep; @@ -323,6 +324,50 @@ public void testAllocateActionOnlyReplicas() throws Exception { }); } + public void testWaitForSnapshot() throws Exception { + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + createNewSingletonPolicy("delete", new WaitForSnapshotAction("slm")); + updatePolicy(index, policy); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getName(), equalTo("wait-for-snapshot"))); + assertBusy(() -> assertThat(getFailedStepForIndex(index), equalTo("wait-for-snapshot"))); + + createSnapshotRepo(); + createSlmPolicy(); + + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); + + Request request = new Request("PUT", "/_slm/policy/slm/_execute"); + assertOK(client().performRequest(request)); + + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed"))); + } + + public void testWaitForSnapshotSlmExecutedBefore() throws Exception { + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + createNewSingletonPolicy("delete", new WaitForSnapshotAction("slm")); + + createSnapshotRepo(); + createSlmPolicy(); + + Request request = new Request("PUT", "/_slm/policy/slm/_execute"); + assertOK(client().performRequest(request)); + + updatePolicy(index, policy); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getName(), equalTo("wait-for-snapshot"))); + + request = new Request("PUT", "/_slm/policy/slm/_execute"); + assertOK(client().performRequest(request)); + + request = new Request("PUT", "/_slm/policy/slm/_execute"); + assertOK(client().performRequest(request)); + + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed"))); + } + public void testDelete() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); @@ -1586,4 +1631,35 @@ private String getSnapshotState(String snapshot) throws IOException { assertThat(snapResponse.get("snapshot"), equalTo(snapshot)); return (String) snapResponse.get("state"); } + + private void createSlmPolicy() throws IOException { + Request request; + request = new Request("PUT", "/_slm/policy/slm"); + request.setJsonEntity(Strings + .toString(JsonXContent.contentBuilder() + .startObject() + .field("schedule", "59 59 23 31 12 ? 2099") + .field("repository", "repo") + .field("name", "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT)) + .startObject("config") + .endObject() + .endObject())); + + assertOK(client().performRequest(request)); + } + + private void createSnapshotRepo() throws IOException { + Request request = new Request("PUT", "/_snapshot/repo"); + request.setJsonEntity(Strings + .toString(JsonXContent.contentBuilder() + .startObject() + .field("type", "fs") + .startObject("settings") + .field("compress", randomBoolean()) + .field("location", System.getProperty("tests.path.repo")) + .field("max_snapshot_bytes_per_sec", "256b") + .endObject() + .endObject())); + assertOK(client().performRequest(request)); + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 947d1752dad65..4b64e306d8f7f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -244,7 +245,8 @@ public List getNa new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SetPriorityAction.NAME), SetPriorityAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(WaitForSnapshotAction.NAME), WaitForSnapshotAction::parse) ); } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java index cb7ace64b10a3..08f392b4f875c 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import java.io.IOException; import java.util.ArrayList; @@ -80,6 +81,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), @@ -99,6 +101,8 @@ protected NamedXContentRegistry xContentRegistry() { (p) -> TimeseriesLifecycleType.INSTANCE), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(WaitForSnapshotAction.NAME), WaitForSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), From 323bbb835632daae0de6e0de6b0d5121487d290b Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 9 Jan 2020 21:09:23 +0100 Subject: [PATCH 2/6] Fix flaky TimeSeriesLifecycleActionsIT#testWaitForSnapshot test This change adds some randomness and cleanup step to TimeSeriesLifecycleActionsIT#testWaitForSnapshot and testWaitForSnapshotSlmExecutedBefore tests in attempt to make them stable. Reletes to #50781 --- .../ilm/TimeSeriesLifecycleActionsIT.java | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 6d11c151e4e7e..4b1771d374774 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -327,45 +327,58 @@ public void testAllocateActionOnlyReplicas() throws Exception { public void testWaitForSnapshot() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); - createNewSingletonPolicy("delete", new WaitForSnapshotAction("slm")); + String smlPolicy = randomAlphaOfLengthBetween(4,10); + createNewSingletonPolicy("delete", new WaitForSnapshotAction(smlPolicy)); updatePolicy(index, policy); assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getName(), equalTo("wait-for-snapshot"))); assertBusy(() -> assertThat(getFailedStepForIndex(index), equalTo("wait-for-snapshot"))); - createSnapshotRepo(); - createSlmPolicy(); + String repo = createSnapshotRepo(); + createSlmPolicy(smlPolicy, repo); assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); - Request request = new Request("PUT", "/_slm/policy/slm/_execute"); + Request request = new Request("PUT", "/_slm/policy/"+smlPolicy+"/_execute"); + assertOK(client().performRequest(request)); + + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")),20, TimeUnit.SECONDS); + + request = new Request("DELETE", "/_slm/policy/" + smlPolicy); assertOK(client().performRequest(request)); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed"))); + request = new Request("DELETE", "/_snapshot/" + repo); + assertOK(client().performRequest(request)); } public void testWaitForSnapshotSlmExecutedBefore() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); - createNewSingletonPolicy("delete", new WaitForSnapshotAction("slm")); + String smlPolicy = randomAlphaOfLengthBetween(4, 10); + createNewSingletonPolicy("delete", new WaitForSnapshotAction(smlPolicy)); - createSnapshotRepo(); - createSlmPolicy(); + String repo = createSnapshotRepo(); + createSlmPolicy(smlPolicy, repo); - Request request = new Request("PUT", "/_slm/policy/slm/_execute"); + Request request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); updatePolicy(index, policy); assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); assertBusy(() -> assertThat(getStepKeyForIndex(index).getName(), equalTo("wait-for-snapshot"))); - - request = new Request("PUT", "/_slm/policy/slm/_execute"); + + request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); - request = new Request("PUT", "/_slm/policy/slm/_execute"); + request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed"))); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 20, TimeUnit.SECONDS); + + request = new Request("DELETE", "/_slm/policy/" + smlPolicy); + assertOK(client().performRequest(request)); + + request = new Request("DELETE", "/_snapshot/" + repo); + assertOK(client().performRequest(request)); } public void testDelete() throws Exception { @@ -1632,14 +1645,14 @@ private String getSnapshotState(String snapshot) throws IOException { return (String) snapResponse.get("state"); } - private void createSlmPolicy() throws IOException { + private void createSlmPolicy(String smlPolicy, String repo) throws IOException { Request request; - request = new Request("PUT", "/_slm/policy/slm"); + request = new Request("PUT", "/_slm/policy/" + smlPolicy); request.setJsonEntity(Strings .toString(JsonXContent.contentBuilder() .startObject() .field("schedule", "59 59 23 31 12 ? 2099") - .field("repository", "repo") + .field("repository", repo) .field("name", "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT)) .startObject("config") .endObject() @@ -1648,8 +1661,9 @@ private void createSlmPolicy() throws IOException { assertOK(client().performRequest(request)); } - private void createSnapshotRepo() throws IOException { - Request request = new Request("PUT", "/_snapshot/repo"); + private String createSnapshotRepo() throws IOException { + String repo = randomAlphaOfLengthBetween(4, 10); + Request request = new Request("PUT", "/_snapshot/" + repo); request.setJsonEntity(Strings .toString(JsonXContent.contentBuilder() .startObject() @@ -1661,5 +1675,6 @@ private void createSnapshotRepo() throws IOException { .endObject() .endObject())); assertOK(client().performRequest(request)); + return repo; } } From 9a83f8229000e97c68410ffca75c7158b08c2d96 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 9 Jan 2020 21:15:35 +0100 Subject: [PATCH 3/6] Formatting changes --- .../xpack/ilm/TimeSeriesLifecycleActionsIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 4b1771d374774..c6d4ac3ed564e 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -40,12 +40,12 @@ import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.ShrinkStep; -import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.Step; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.ilm.TerminalPolicyStep; import org.elasticsearch.xpack.core.ilm.UpdateRolloverLifecycleDateStep; import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.hamcrest.Matchers; import org.junit.Before; @@ -327,7 +327,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { public void testWaitForSnapshot() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); - String smlPolicy = randomAlphaOfLengthBetween(4,10); + String smlPolicy = randomAlphaOfLengthBetween(4, 10); createNewSingletonPolicy("delete", new WaitForSnapshotAction(smlPolicy)); updatePolicy(index, policy); assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); @@ -338,10 +338,10 @@ public void testWaitForSnapshot() throws Exception { assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("wait_for_snapshot"))); - Request request = new Request("PUT", "/_slm/policy/"+smlPolicy+"/_execute"); + Request request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")),20, TimeUnit.SECONDS); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 20, TimeUnit.SECONDS); request = new Request("DELETE", "/_slm/policy/" + smlPolicy); assertOK(client().performRequest(request)); From a3ff28e6d77a48615c67c6e504d9968c014d0c1f Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 9 Jan 2020 23:50:47 +0100 Subject: [PATCH 4/6] Longer timeout --- .../xpack/ilm/TimeSeriesLifecycleActionsIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index c6d4ac3ed564e..ce247369e3743 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -341,7 +341,8 @@ public void testWaitForSnapshot() throws Exception { Request request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 20, TimeUnit.SECONDS); + + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 2, TimeUnit.MINUTES); request = new Request("DELETE", "/_slm/policy/" + smlPolicy); assertOK(client().performRequest(request)); @@ -372,7 +373,7 @@ public void testWaitForSnapshotSlmExecutedBefore() throws Exception { request = new Request("PUT", "/_slm/policy/" + smlPolicy + "/_execute"); assertOK(client().performRequest(request)); - assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 20, TimeUnit.SECONDS); + assertBusy(() -> assertThat(getStepKeyForIndex(index).getAction(), equalTo("completed")), 2, TimeUnit.MINUTES); request = new Request("DELETE", "/_slm/policy/" + smlPolicy); assertOK(client().performRequest(request)); @@ -1655,6 +1656,7 @@ private void createSlmPolicy(String smlPolicy, String repo) throws IOException { .field("repository", repo) .field("name", "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT)) .startObject("config") + .field("include_global_state", false) .endObject() .endObject())); From 88efad2046c9a5a90403a73ae4d4183b91078e76 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 13 Jan 2020 23:55:46 +0100 Subject: [PATCH 5/6] Fix Map.of in Java8 --- .../xpack/core/ilm/WaitForSnapshotStepTests.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java index 6c1822235ec97..06b113d55081e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; import java.io.IOException; +import java.util.Collections; import java.util.Map; public class WaitForSnapshotStepTests extends AbstractStepTestCase { @@ -57,7 +58,7 @@ protected WaitForSnapshotStep copyInstance(WaitForSnapshotStep instance) { public void testNoSlmPolicies() { IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(randomLong()))) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Collections.singletonMap("phase_time", Long.toString(randomLong()))) .settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); ImmutableOpenMap.Builder indices = @@ -76,12 +77,12 @@ public void testSlmPolicyNotExecuted() throws IOException { .setModifiedDate(randomLong()) .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) .build(); - SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Collections.singletonMap(instance.getPolicy(), slmPolicy), OperationMode.RUNNING, null); IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(randomLong()))) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Collections.singletonMap("phase_time", Long.toString(randomLong()))) .settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); ImmutableOpenMap.Builder indices = @@ -102,11 +103,11 @@ public void testSlmPolicyExecutedBeforeStep() throws IOException { .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) .setLastSuccess(new SnapshotInvocationRecord("", phaseTime - 10, "")) .build(); - SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Collections.singletonMap(instance.getPolicy(), slmPolicy), OperationMode.RUNNING, null); IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(phaseTime))) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Collections.singletonMap("phase_time", Long.toString(phaseTime))) .settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); ImmutableOpenMap.Builder indices = @@ -127,11 +128,11 @@ public void testSlmPolicyExecutedAfterStep() throws IOException { .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) .setLastSuccess(new SnapshotInvocationRecord("", phaseTime + 10, "")) .build(); - SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Map.of(instance.getPolicy(), slmPolicy), + SnapshotLifecycleMetadata smlMetaData = new SnapshotLifecycleMetadata(Collections.singletonMap(instance.getPolicy(), slmPolicy), OperationMode.RUNNING, null); IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("phase_time", Long.toString(phaseTime))) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Collections.singletonMap("phase_time", Long.toString(phaseTime))) .settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); ImmutableOpenMap.Builder indices = From 55e3d714526ff75f0d0c9f18ea910493240f6994 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 14 Jan 2020 00:04:33 +0100 Subject: [PATCH 6/6] Unused import removed --- .../elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java index 06b113d55081e..6ca711c9f2421 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.Map; public class WaitForSnapshotStepTests extends AbstractStepTestCase {