Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add retention to Snapshot Lifecycle Management #46407

Merged
merged 36 commits into from
Sep 9, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
23cfe40
Add base framework for snapshot retention (#43605)
dakrone Jun 27, 2019
3c1f331
Add SnapshotRetentionConfiguration for retention configuration (#43777)
dakrone Jul 15, 2019
7288987
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 22, 2019
73c8f3a
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 22, 2019
f520f42
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 23, 2019
c30265a
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 24, 2019
480800a
Implement SnapshotRetentionTask's snapshot filtering and delet… (#44764)
dakrone Jul 25, 2019
73c5287
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 26, 2019
646f5bb
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 29, 2019
d3775fc
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 30, 2019
67ff2ef
Allow empty/missing SLM retention configuration (#45018)
dakrone Jul 31, 2019
337a51c
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Jul 31, 2019
ae6332d
Add min_count and max_count as SLM retention predicates (#44926)
dakrone Jul 31, 2019
5faa8f9
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 1, 2019
b442673
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 6, 2019
445aca9
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 6, 2019
852106c
Time-bound deletion of snapshots in retention delete function (#45065)
dakrone Aug 7, 2019
6a35e57
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 7, 2019
4be8725
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 8, 2019
47c6553
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 13, 2019
9d17b78
Add SLM metrics gathering and endpoint (#45362)
dakrone Aug 13, 2019
c2b13df
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 14, 2019
13f7602
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 20, 2019
f42a478
Record history of SLM retention actions (#45513)
gwbrown Aug 21, 2019
fcd0a30
Merge branch 'master' into slm-retention
gwbrown Aug 21, 2019
2d56ae2
Retry SLM retention after currently running snapshot completes (#45802)
dakrone Aug 22, 2019
be684f8
[TEST] Be less strict for stats checking
dakrone Aug 22, 2019
d534689
Skip SLM retention if ILM is STOPPING or STOPPED (#45869)
dakrone Aug 22, 2019
9e44680
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 23, 2019
3498705
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Aug 29, 2019
06e2d69
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Sep 3, 2019
7129832
Check all actions preventing snapshot delete during retention (#45992)
dakrone Sep 3, 2019
4a164fc
Expose SLM policy stats in get SLM policy API (#45989)
dakrone Sep 3, 2019
5eead86
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Sep 4, 2019
07395f3
Rewrite SnapshotLifecycleIT as as ESIntegTestCase (#46356)
dakrone Sep 5, 2019
598ca6d
Merge remote-tracking branch 'origin/master' into slm-retention
dakrone Sep 5, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyResponse;
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyRequest;
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyResponse;
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsRequest;
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsResponse;
import org.elasticsearch.client.slm.PutSnapshotLifecyclePolicyRequest;

import java.io.IOException;
Expand Down Expand Up @@ -464,4 +466,40 @@ public Cancellable executeSnapshotLifecyclePolicyAsync(
request, IndexLifecycleRequestConverters::executeSnapshotLifecyclePolicy,
options, ExecuteSnapshotLifecyclePolicyResponse::fromXContent, listener, emptySet());
}

/**
* Retrieve snapshot lifecycle statistics.
* See <pre>
* https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/
* java-rest-high-ilm-slm-get-snapshot-lifecycle-stats.html
* </pre>
* for more.
* @param request the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public GetSnapshotLifecycleStatsResponse getSnapshotLifecycleStats(GetSnapshotLifecycleStatsRequest request,
RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::getSnapshotLifecycleStats,
options, GetSnapshotLifecycleStatsResponse::fromXContent, emptySet());
}

/**
* Asynchronously retrieve snapshot lifecycle statistics.
* See <pre>
* https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/
* java-rest-high-ilm-slm-get-snapshot-lifecycle-stats.html
* </pre>
* for more.
* @param request the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener the listener to be notified upon request completion
* @return cancellable that may be used to cancel the request
*/
public Cancellable getSnapshotLifecycleStatsAsync(GetSnapshotLifecycleStatsRequest request, RequestOptions options,
ActionListener<GetSnapshotLifecycleStatsResponse> listener) {
return restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::getSnapshotLifecycleStats,
options, GetSnapshotLifecycleStatsResponse::fromXContent, listener, emptySet());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.elasticsearch.client.slm.DeleteSnapshotLifecyclePolicyRequest;
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyRequest;
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyRequest;
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsRequest;
import org.elasticsearch.client.slm.PutSnapshotLifecyclePolicyRequest;
import org.elasticsearch.common.Strings;

Expand Down Expand Up @@ -215,4 +216,14 @@ static Request executeSnapshotLifecyclePolicy(ExecuteSnapshotLifecyclePolicyRequ
request.addParameters(params.asMap());
return request;
}

static Request getSnapshotLifecycleStats(GetSnapshotLifecycleStatsRequest getSnapshotLifecycleStatsRequest) {
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_slm/stats").build();
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
RequestConverters.Params params = new RequestConverters.Params();
params.withMasterTimeout(getSnapshotLifecycleStatsRequest.masterNodeTimeout());
params.withTimeout(getSnapshotLifecycleStatsRequest.timeout());
request.addParameters(params.asMap());
return request;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.client.slm;

import org.elasticsearch.client.TimedRequest;

public class GetSnapshotLifecycleStatsRequest extends TimedRequest {

public GetSnapshotLifecycleStatsRequest() {
super();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.client.slm;

import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;

import java.io.IOException;
import java.util.Objects;

public class GetSnapshotLifecycleStatsResponse implements ToXContentObject {

private final SnapshotLifecycleStats stats;

public GetSnapshotLifecycleStatsResponse(SnapshotLifecycleStats stats) {
this.stats = stats;
}

public SnapshotLifecycleStats getStats() {
return this.stats;
}

public static GetSnapshotLifecycleStatsResponse fromXContent(XContentParser parser) throws IOException {
return new GetSnapshotLifecycleStatsResponse(SnapshotLifecycleStats.parse(parser));
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return stats.toXContent(builder, params);
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}

if (o == null || getClass() != o.getClass()) {
return false;
}

GetSnapshotLifecycleStatsResponse other = (GetSnapshotLifecycleStatsResponse) o;
return Objects.equals(this.stats, other.stats);
}

@Override
public int hashCode() {
return Objects.hash(this.stats);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ public class SnapshotLifecyclePolicy implements ToXContentObject {
private final String schedule;
private final String repository;
private final Map<String, Object> configuration;
private final SnapshotRetentionConfiguration retentionPolicy;

private static final ParseField NAME = new ParseField("name");
private static final ParseField SCHEDULE = new ParseField("schedule");
private static final ParseField REPOSITORY = new ParseField("repository");
private static final ParseField CONFIG = new ParseField("config");
private static final ParseField RETENTION = new ParseField("retention");

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<SnapshotLifecyclePolicy, String> PARSER =
Expand All @@ -52,23 +54,27 @@ public class SnapshotLifecyclePolicy implements ToXContentObject {
String schedule = (String) a[1];
String repo = (String) a[2];
Map<String, Object> config = (Map<String, Object>) a[3];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config);
SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention);
});

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
PARSER.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
PARSER.declareString(ConstructingObjectParser.constructorArg(), REPOSITORY);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), CONFIG);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), SnapshotRetentionConfiguration::parse, RETENTION);
}

public SnapshotLifecyclePolicy(final String id, final String name, final String schedule,
final String repository, @Nullable Map<String, Object> configuration) {
this.id = Objects.requireNonNull(id);
this.name = name;
this.schedule = schedule;
this.repository = repository;
final String repository, @Nullable final Map<String, Object> configuration,
@Nullable final SnapshotRetentionConfiguration retentionPolicy) {
this.id = Objects.requireNonNull(id, "policy id is required");
this.name = Objects.requireNonNull(name, "policy snapshot name is required");
this.schedule = Objects.requireNonNull(schedule, "policy schedule is required");
this.repository = Objects.requireNonNull(repository, "policy snapshot repository is required");
this.configuration = configuration;
this.retentionPolicy = retentionPolicy;
}

public String getId() {
Expand All @@ -92,6 +98,11 @@ public Map<String, Object> getConfig() {
return this.configuration;
}

@Nullable
public SnapshotRetentionConfiguration getRetentionPolicy() {
return this.retentionPolicy;
}

public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) {
return PARSER.apply(parser, id);
}
Expand All @@ -105,13 +116,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (this.configuration != null) {
builder.field(CONFIG.getPreferredName(), this.configuration);
}
if (this.retentionPolicy != null) {
builder.field(RETENTION.getPreferredName(), this.retentionPolicy);
}
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(id, name, schedule, repository, configuration);
return Objects.hash(id, name, schedule, repository, configuration, retentionPolicy);
}

@Override
Expand All @@ -128,7 +142,8 @@ public boolean equals(Object obj) {
Objects.equals(name, other.name) &&
Objects.equals(schedule, other.schedule) &&
Objects.equals(repository, other.repository) &&
Objects.equals(configuration, other.configuration);
Objects.equals(configuration, other.configuration) &&
Objects.equals(retentionPolicy, other.retentionPolicy);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public class SnapshotLifecyclePolicyMetadata implements ToXContentObject {
static final ParseField NEXT_EXECUTION_MILLIS = new ParseField("next_execution_millis");
static final ParseField NEXT_EXECUTION = new ParseField("next_execution");
static final ParseField SNAPSHOT_IN_PROGRESS = new ParseField("in_progress");
static final ParseField POLICY_STATS = new ParseField("stats");

private final SnapshotLifecyclePolicy policy;
private final long version;
Expand All @@ -53,6 +54,7 @@ public class SnapshotLifecyclePolicyMetadata implements ToXContentObject {
private final SnapshotInvocationRecord lastFailure;
@Nullable
private final SnapshotInProgress snapshotInProgress;
private final SnapshotLifecycleStats.SnapshotPolicyStats policyStats;

@SuppressWarnings("unchecked")
public static final ConstructingObjectParser<SnapshotLifecyclePolicyMetadata, String> PARSER =
Expand All @@ -65,8 +67,9 @@ public class SnapshotLifecyclePolicyMetadata implements ToXContentObject {
SnapshotInvocationRecord lastFailure = (SnapshotInvocationRecord) a[4];
long nextExecution = (long) a[5];
SnapshotInProgress sip = (SnapshotInProgress) a[6];

return new SnapshotLifecyclePolicyMetadata(policy, version, modifiedDate, lastSuccess, lastFailure, nextExecution, sip);
SnapshotLifecycleStats.SnapshotPolicyStats stats = (SnapshotLifecycleStats.SnapshotPolicyStats) a[7];
return new SnapshotLifecyclePolicyMetadata(policy, version, modifiedDate, lastSuccess,
lastFailure, nextExecution, sip, stats);
});

static {
Expand All @@ -77,6 +80,9 @@ public class SnapshotLifecyclePolicyMetadata implements ToXContentObject {
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), SnapshotInvocationRecord::parse, LAST_FAILURE);
PARSER.declareLong(ConstructingObjectParser.constructorArg(), NEXT_EXECUTION_MILLIS);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), SnapshotInProgress::parse, SNAPSHOT_IN_PROGRESS);
PARSER.declareObject(ConstructingObjectParser.constructorArg(),
(p, c) -> SnapshotLifecycleStats.SnapshotPolicyStats.parse(p, "policy"), POLICY_STATS);

}

public static SnapshotLifecyclePolicyMetadata parse(XContentParser parser, String id) {
Expand All @@ -86,14 +92,16 @@ public static SnapshotLifecyclePolicyMetadata parse(XContentParser parser, Strin
public SnapshotLifecyclePolicyMetadata(SnapshotLifecyclePolicy policy, long version, long modifiedDate,
SnapshotInvocationRecord lastSuccess, SnapshotInvocationRecord lastFailure,
long nextExecution,
@Nullable SnapshotInProgress snapshotInProgress) {
@Nullable SnapshotInProgress snapshotInProgress,
SnapshotLifecycleStats.SnapshotPolicyStats policyStats) {
this.policy = policy;
this.version = version;
this.modifiedDate = modifiedDate;
this.lastSuccess = lastSuccess;
this.lastFailure = lastFailure;
this.nextExecution = nextExecution;
this.snapshotInProgress = snapshotInProgress;
this.policyStats = policyStats;
}

public SnapshotLifecyclePolicy getPolicy() {
Expand Down Expand Up @@ -124,6 +132,10 @@ public long getNextExecution() {
return this.nextExecution;
}

public SnapshotLifecycleStats.SnapshotPolicyStats getPolicyStats() {
return this.policyStats;
}

@Nullable
public SnapshotInProgress getSnapshotInProgress() {
return this.snapshotInProgress;
Expand All @@ -145,13 +157,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (snapshotInProgress != null) {
builder.field(SNAPSHOT_IN_PROGRESS.getPreferredName(), snapshotInProgress);
}
builder.startObject(POLICY_STATS.getPreferredName());
this.policyStats.toXContent(builder, params);
builder.endObject();
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(policy, version, modifiedDate, lastSuccess, lastFailure, nextExecution);
return Objects.hash(policy, version, modifiedDate, lastSuccess, lastFailure, nextExecution, policyStats);
}

@Override
Expand All @@ -168,7 +183,8 @@ public boolean equals(Object obj) {
Objects.equals(modifiedDate, other.modifiedDate) &&
Objects.equals(lastSuccess, other.lastSuccess) &&
Objects.equals(lastFailure, other.lastFailure) &&
Objects.equals(nextExecution, other.nextExecution);
Objects.equals(nextExecution, other.nextExecution) &&
Objects.equals(policyStats, other.policyStats);
}

public static class SnapshotInProgress implements ToXContentObject {
Expand Down
Loading