From 6eb4600e93ac1ee3ac027c3cdad9823b43048bf8 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 5 Jun 2019 17:30:31 -0600 Subject: [PATCH] Add custom metadata to snapshots (#41281) Adds a metadata field to snapshots which can be used to store arbitrary key-value information. This may be useful for attaching a description of why a snapshot was taken, tagging snapshots to make categorization easier, or identifying the source of automatically-created snapshots. --- .../org/elasticsearch/client/SnapshotIT.java | 48 ++++++ docs/reference/modules/snapshots.asciidoc | 9 +- .../test/snapshot.get/10_basic.yml | 39 +++++ .../create/CreateSnapshotRequest.java | 53 ++++++- .../snapshots/get/GetSnapshotsResponse.java | 6 + .../cluster/SnapshotsInProgress.java | 30 +++- .../repositories/FilterRepository.java | 6 +- .../repositories/Repository.java | 4 +- .../blobstore/BlobStoreRepository.java | 5 +- .../elasticsearch/snapshots/SnapshotInfo.java | 71 +++++++-- .../snapshots/SnapshotsService.java | 11 +- .../create/CreateSnapshotRequestTests.java | 63 ++++++++ .../create/CreateSnapshotResponseTests.java | 15 +- .../get/GetSnapshotsResponseTests.java | 19 ++- .../cluster/ClusterStateDiffIT.java | 14 +- .../cluster/SnapshotsInProgressTests.java | 3 +- .../MetaDataDeleteIndexServiceTests.java | 4 +- .../MetaDataIndexStateServiceTests.java | 4 +- .../RepositoriesServiceTests.java | 3 +- .../SharedClusterSnapshotRestoreIT.java | 3 +- .../snapshots/SnapshotInfoTests.java | 149 ++++++++++++++++++ ...SnapshotsInProgressSerializationTests.java | 3 +- .../index/shard/RestoreOnlyRepository.java | 2 +- .../xpack/ccr/repository/CcrRepository.java | 3 +- 24 files changed, 518 insertions(+), 49 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 616850c513af7..5c30de5c0572c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -41,9 +41,13 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.SnapshotInfo; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; @@ -139,6 +143,9 @@ public void testCreateSnapshot() throws IOException { CreateSnapshotRequest request = new CreateSnapshotRequest(repository, snapshot); boolean waitForCompletion = randomBoolean(); request.waitForCompletion(waitForCompletion); + if (randomBoolean()) { + request.userMetadata(randomUserMetadata()); + } request.partial(randomBoolean()); request.includeGlobalState(randomBoolean()); @@ -167,6 +174,8 @@ public void testGetSnapshots() throws IOException { CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1); CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2); createSnapshotRequest2.waitForCompletion(true); + Map originalMetadata = randomUserMetadata(); + createSnapshotRequest2.userMetadata(originalMetadata); CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2); // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. assertEquals(RestStatus.OK, putSnapshotResponse1.status()); @@ -186,6 +195,15 @@ public void testGetSnapshots() throws IOException { assertEquals(2, response.getSnapshots().size()); assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()), contains("test_snapshot1", "test_snapshot2")); + Optional> returnedMetadata = response.getSnapshots().stream() + .filter(s -> s.snapshotId().getName().equals("test_snapshot2")) + .findFirst() + .map(SnapshotInfo::userMetadata); + if (returnedMetadata.isPresent()) { + assertEquals(originalMetadata, returnedMetadata.get()); + } else { + assertNull("retrieved metadata is null, expected non-null metadata", originalMetadata); + } } public void testSnapshotsStatus() throws IOException { @@ -231,6 +249,9 @@ public void testRestoreSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot); createSnapshotRequest.indices(testIndex); createSnapshotRequest.waitForCompletion(true); + if (randomBoolean()) { + createSnapshotRequest.userMetadata(randomUserMetadata()); + } CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest); assertEquals(RestStatus.OK, createSnapshotResponse.status()); @@ -261,6 +282,9 @@ public void testDeleteSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repository, snapshot); createSnapshotRequest.waitForCompletion(true); + if (randomBoolean()) { + createSnapshotRequest.userMetadata(randomUserMetadata()); + } CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest); // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. assertEquals(RestStatus.OK, createSnapshotResponse.status()); @@ -270,4 +294,28 @@ public void testDeleteSnapshot() throws IOException { assertTrue(response.isAcknowledged()); } + + private static Map randomUserMetadata() { + if (randomBoolean()) { + return null; + } + + Map metadata = new HashMap<>(); + long fields = randomLongBetween(0, 4); + for (int i = 0; i < fields; i++) { + if (randomBoolean()) { + metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), + randomAlphaOfLengthBetween(5, 5)); + } else { + Map nested = new HashMap<>(); + long nestedFields = randomLongBetween(0, 4); + for (int j = 0; j < nestedFields; j++) { + nested.put(randomValueOtherThanMany(nested::containsKey, () -> randomAlphaOfLengthBetween(2,10)), + randomAlphaOfLengthBetween(5, 5)); + } + metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), nested); + } + } + return metadata; + } } diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index d5d1e44166034..9a33cdd7214a5 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -349,7 +349,11 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true { "indices": "index_1,index_2", "ignore_unavailable": true, - "include_global_state": false + "include_global_state": false, + "_meta": { + "taken_by": "kimchy", + "taken_because": "backup before upgrading" + } } ----------------------------------- // CONSOLE @@ -363,6 +367,9 @@ By setting `include_global_state` to false it's possible to prevent the cluster the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have all primary shards available. This behaviour can be changed by setting `partial` to `true`. +The `_meta` field can be used to attach arbitrary metadata to the snapshot. This may be a record of who took the snapshot, +why it was taken, or any other data that might be useful. + Snapshot names can be automatically derived using <>, similarly as when creating new indices. Note that special characters need to be URI encoded. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index aa15ca34ff0af..00656be2b59f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -87,6 +87,7 @@ setup: - is_false: snapshots.0.failures - is_false: snapshots.0.shards - is_false: snapshots.0.version + - is_false: snapshots.0._meta - do: snapshot.delete: @@ -152,3 +153,41 @@ setup: snapshot.delete: repository: test_repo_get_1 snapshot: test_snapshot_without_include_global_state + +--- +"Get snapshot info with metadata": + - skip: + version: " - 7.9.99" + reason: "https://github.com/elastic/elasticsearch/pull/41281 not yet backported to 7.x" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata + wait_for_completion: true + body: | + { "metadata": {"taken_by": "test", "foo": {"bar": "baz"}} } + + - do: + snapshot.get: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot_with_metadata } + - match: { snapshots.0.state: SUCCESS } + - match: { snapshots.0.metadata.taken_by: test } + - match: { snapshots.0.metadata.foo.bar: baz } + + - do: + snapshot.delete: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 15fbac35bffd2..a72120e328b00 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -19,12 +19,14 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -46,6 +48,7 @@ import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; +import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED; /** * Create snapshot request @@ -63,6 +66,7 @@ */ public class CreateSnapshotRequest extends MasterNodeRequest implements IndicesRequest.Replaceable, ToXContentObject { + public static int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily private String snapshot; @@ -80,6 +84,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest userMetadata; + public CreateSnapshotRequest() { } @@ -104,6 +110,9 @@ public CreateSnapshotRequest(StreamInput in) throws IOException { includeGlobalState = in.readBoolean(); waitForCompletion = in.readBoolean(); partial = in.readBoolean(); + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } } @Override @@ -117,6 +126,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(includeGlobalState); out.writeBoolean(waitForCompletion); out.writeBoolean(partial); + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(userMetadata); + } } @Override @@ -144,9 +156,28 @@ public ActionRequestValidationException validate() { if (settings == null) { validationException = addValidationError("settings is null", validationException); } + final int metadataSize = metadataSize(userMetadata); + if (metadataSize > MAXIMUM_METADATA_BYTES) { + validationException = addValidationError("metadata must be smaller than 1024 bytes, but was [" + metadataSize + "]", + validationException); + } return validationException; } + private static int metadataSize(Map userMetadata) { + if (userMetadata == null) { + return 0; + } + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.value(userMetadata); + int size = BytesReference.bytes(builder).length(); + return size; + } catch (IOException e) { + // This should not be possible as we are just rendering the xcontent in memory + throw new ElasticsearchException(e); + } + } + /** * Sets the snapshot name * @@ -378,6 +409,15 @@ public boolean includeGlobalState() { return includeGlobalState; } + public Map userMetadata() { + return userMetadata; + } + + public CreateSnapshotRequest userMetadata(Map userMetadata) { + this.userMetadata = userMetadata; + return this; + } + /** * Parses snapshot definition. * @@ -405,6 +445,11 @@ public CreateSnapshotRequest source(Map source) { settings((Map) entry.getValue()); } else if (name.equals("include_global_state")) { includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state"); + } else if (name.equals("metadata")) { + if (entry.getValue() != null && (entry.getValue() instanceof Map == false)) { + throw new IllegalArgumentException("malformed metadata, should be an object"); + } + userMetadata((Map) entry.getValue()); } } indicesOptions(IndicesOptions.fromMap(source, indicesOptions)); @@ -433,6 +478,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indicesOptions != null) { indicesOptions.toXContent(builder, params); } + builder.field("metadata", userMetadata); builder.endObject(); return builder; } @@ -460,12 +506,14 @@ public boolean equals(Object o) { Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(settings, that.settings) && - Objects.equals(masterNodeTimeout, that.masterNodeTimeout); + Objects.equals(masterNodeTimeout, that.masterNodeTimeout) && + Objects.equals(userMetadata, that.userMetadata); } @Override public int hashCode() { - int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState, waitForCompletion); + int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState, + waitForCompletion, userMetadata); result = 31 * result + Arrays.hashCode(indices); return result; } @@ -482,6 +530,7 @@ public String toString() { ", includeGlobalState=" + includeGlobalState + ", waitForCompletion=" + waitForCompletion + ", masterNodeTimeout=" + masterNodeTimeout + + ", metadata=" + userMetadata + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 6f757cb60ca86..62ddc5b7d9df3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -117,4 +118,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(snapshots); } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 26560f92c5dad..b933c856fe04e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -44,6 +44,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED; + /** * Meta data about snapshots that are currently executing */ @@ -84,11 +86,12 @@ public static class Entry { private final ImmutableOpenMap> waitingIndices; private final long startTime; private final long repositoryStateId; + @Nullable private final Map userMetadata; @Nullable private final String failure; public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, long repositoryStateId, ImmutableOpenMap shards, - String failure) { + String failure, Map userMetadata) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; @@ -104,21 +107,23 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta } this.repositoryStateId = repositoryStateId; this.failure = failure; + this.userMetadata = userMetadata; } public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - long startTime, long repositoryStateId, ImmutableOpenMap shards) { - this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null); + long startTime, long repositoryStateId, ImmutableOpenMap shards, + Map userMetadata) { + this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, entry.failure); + entry.repositoryStateId, shards, entry.failure, entry.userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards, String failure) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, failure); + entry.repositoryStateId, shards, failure, entry.userMetadata); } public Entry(Entry entry, ImmutableOpenMap shards) { @@ -149,6 +154,10 @@ public boolean includeGlobalState() { return includeGlobalState; } + public Map userMetadata() { + return userMetadata; + } + public boolean partial() { return partial; } @@ -433,6 +442,10 @@ public SnapshotsInProgress(StreamInput in) throws IOException { } else { failure = null; } + Map userMetadata = null; + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } entries[i] = new Entry(snapshot, includeGlobalState, partial, @@ -441,7 +454,9 @@ public SnapshotsInProgress(StreamInput in) throws IOException { startTime, repositoryStateId, builder.build(), - failure); + failure, + userMetadata + ); } this.entries = Arrays.asList(entries); } @@ -473,6 +488,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeOptionalString(entry.failure); } + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(entry.userMetadata); + } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 1fa42579617e1..8c9eff0698835 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; public class FilterRepository implements Repository { @@ -79,9 +80,10 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met @Override public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List indices, long startTime, String failure, int totalShards, - List shardFailures, long repositoryStateId, boolean includeGlobalState) { + List shardFailures, long repositoryStateId, boolean includeGlobalState, + Map userMetadata) { return in.finalizeSnapshot(snapshotId, indices, startTime, failure, totalShards, shardFailures, repositoryStateId, - includeGlobalState); + includeGlobalState, userMetadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 3aa19cb130cae..1d828da344bb6 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.function.Function; /** @@ -135,7 +136,8 @@ default Repository create(RepositoryMetaData metaData, Function indices, long startTime, String failure, int totalShards, - List shardFailures, long repositoryStateId, boolean includeGlobalState); + List shardFailures, long repositoryStateId, boolean includeGlobalState, + Map userMetadata); /** * Deletes snapshot diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 67ea26616ac42..a74e4a51f4cf9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -519,11 +519,12 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final int totalShards, final List shardFailures, final long repositoryStateId, - final boolean includeGlobalState) { + final boolean includeGlobalState, + final Map userMetadata) { SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, System.currentTimeMillis(), totalShards, shardFailures, - includeGlobalState); + includeGlobalState, userMetadata); try { snapshotFormat.write(blobStoreSnapshot, blobContainer(), snapshotId.getUUID()); final RepositoryData repositoryData = getRepositoryData(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 8b907a54e51cb..d6e889c599f17 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -51,6 +52,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, public static final String CONTEXT_MODE_PARAM = "context_mode"; public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT"; + public static final Version METADATA_FIELD_INTRODUCED = Version.V_7_3_0; private static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("strictDateOptionalTime"); private static final String SNAPSHOT = "snapshot"; private static final String UUID = "uuid"; @@ -74,6 +76,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String TOTAL_SHARDS = "total_shards"; private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; + private static final String USER_METADATA = "metadata"; private static final Version INCLUDE_GLOBAL_STATE_INTRODUCED = Version.V_6_2_0; @@ -90,6 +93,7 @@ public static final class SnapshotInfoBuilder { private long endTime = 0L; private ShardStatsBuilder shardStatsBuilder = null; private Boolean includeGlobalState = null; + private Map userMetadata = null; private int version = -1; private List shardFailures = null; @@ -129,6 +133,10 @@ private void setIncludeGlobalState(Boolean includeGlobalState) { this.includeGlobalState = includeGlobalState; } + private void setUserMetadata(Map userMetadata) { + this.userMetadata = userMetadata; + } + private void setVersion(int version) { this.version = version; } @@ -155,7 +163,7 @@ public SnapshotInfo build() { } return new SnapshotInfo(snapshotId, indices, snapshotState, reason, version, startTime, endTime, - totalShards, successfulShards, shardFailures, includeGlobalState); + totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata); } } @@ -196,6 +204,7 @@ int getSuccessfulShards() { SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS)); SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE)); + SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setUserMetadata, (p, c) -> p.map() , new ParseField(USER_METADATA)); SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID)); SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setShardFailures, SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER, new ParseField(FAILURES)); @@ -225,6 +234,9 @@ int getSuccessfulShards() { @Nullable private Boolean includeGlobalState; + @Nullable + private final Map userMetadata; + @Nullable private final Version version; @@ -232,28 +244,30 @@ int getSuccessfulShards() { public SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state) { this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } public SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, Version version) { this(snapshotId, indices, state, null, version, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } - public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, Boolean includeGlobalState) { + public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, Boolean includeGlobalState, + Map userMetadata) { this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, - 0, 0, Collections.emptyList(), includeGlobalState); + 0, 0, Collections.emptyList(), includeGlobalState, userMetadata); } public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, String reason, long endTime, - int totalShards, List shardFailures, Boolean includeGlobalState) { + int totalShards, List shardFailures, Boolean includeGlobalState, + Map userMetadata) { this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT, - startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState); + startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata); } private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, String reason, Version version, long startTime, long endTime, int totalShards, int successfulShards, List shardFailures, - Boolean includeGlobalState) { + Boolean includeGlobalState, Map userMetadata) { this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.state = state; @@ -265,6 +279,7 @@ private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState this.successfulShards = successfulShards; this.shardFailures = Objects.requireNonNull(shardFailures); this.includeGlobalState = includeGlobalState; + this.userMetadata = userMetadata; } /** @@ -298,6 +313,11 @@ public SnapshotInfo(final StreamInput in) throws IOException { if (in.getVersion().onOrAfter(INCLUDE_GLOBAL_STATE_INTRODUCED)) { includeGlobalState = in.readOptionalBoolean(); } + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } else { + userMetadata = null; + } } /** @@ -308,7 +328,7 @@ public static SnapshotInfo incompatible(SnapshotId snapshotId) { return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE, "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown", null, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } /** @@ -432,6 +452,15 @@ public Version version() { return version; } + /** + * Returns the custom metadata that was attached to this snapshot at creation time. + * @return custom metadata + */ + @Nullable + public Map userMetadata() { + return userMetadata; + } + /** * Compares two snapshots by their start time; if the start times are the same, then * compares the two snapshots by their snapshot ids. @@ -496,6 +525,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (includeGlobalState != null) { builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState); } + if (userMetadata != null) { + builder.field(USER_METADATA, userMetadata); + } if (verbose || state != null) { builder.field(STATE, state); } @@ -547,6 +579,7 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final if (includeGlobalState != null) { builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState); } + builder.field(USER_METADATA, userMetadata); builder.field(START_TIME, startTime); builder.field(END_TIME, endTime); builder.field(TOTAL_SHARDS, totalShards); @@ -577,6 +610,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr int totalShards = 0; int successfulShards = 0; Boolean includeGlobalState = null; + Map userMetadata = null; List shardFailures = Collections.emptyList(); if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); @@ -632,8 +666,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr parser.skipChildren(); } } else if (token == XContentParser.Token.START_OBJECT) { - // It was probably created by newer version - ignoring - parser.skipChildren(); + if (USER_METADATA.equals(currentFieldName)) { + userMetadata = parser.map(); + } else { + // It was probably created by newer version - ignoring + parser.skipChildren(); + } } } } @@ -655,7 +693,8 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr totalShards, successfulShards, shardFailures, - includeGlobalState); + includeGlobalState, + userMetadata); } @Override @@ -689,6 +728,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(INCLUDE_GLOBAL_STATE_INTRODUCED)) { out.writeOptionalBoolean(includeGlobalState); } + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(userMetadata); + } } private static SnapshotState snapshotState(final String reason, final List shardFailures) { @@ -718,13 +760,14 @@ public boolean equals(Object o) { Objects.equals(indices, that.indices) && Objects.equals(includeGlobalState, that.includeGlobalState) && Objects.equals(version, that.version) && - Objects.equals(shardFailures, that.shardFailures); + Objects.equals(shardFailures, that.shardFailures) && + Objects.equals(userMetadata, that.userMetadata); } @Override public int hashCode() { return Objects.hash(snapshotId, state, reason, indices, startTime, endTime, - totalShards, successfulShards, includeGlobalState, version, shardFailures); + totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index b1d365f7ff138..1563facd335c0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -287,7 +287,8 @@ public ClusterState execute(ClusterState currentState) { snapshotIndices, System.currentTimeMillis(), repositoryData.getGenId(), - null); + null, + request.userMetadata()); initializingSnapshots.add(newSnapshot.snapshot()); snapshots = new SnapshotsInProgress(newSnapshot); } else { @@ -557,7 +558,8 @@ private void cleanupAfterError(Exception exception) { 0, Collections.emptyList(), snapshot.getRepositoryStateId(), - snapshot.includeGlobalState()); + snapshot.includeGlobalState(), + snapshot.userMetadata()); } catch (Exception inner) { inner.addSuppressed(exception); logger.warn(() -> new ParameterizedMessage("[{}] failed to close snapshot in repository", @@ -572,7 +574,7 @@ private void cleanupAfterError(Exception exception) { private static SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) { return new SnapshotInfo(entry.snapshot().getSnapshotId(), entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), - entry.startTime(), entry.includeGlobalState()); + entry.startTime(), entry.includeGlobalState(), entry.userMetadata()); } /** @@ -988,7 +990,8 @@ protected void doRun() { entry.shards().size(), unmodifiableList(shardFailures), entry.getRepositoryStateId(), - entry.includeGlobalState()); + entry.includeGlobalState(), + entry.userMetadata()); removeSnapshotFromClusterState(snapshot, snapshotInfo, null); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 0b598be6849cb..9f7bd5f6a0149 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -19,10 +19,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions.Option; import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent.MapParams; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,6 +43,10 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.snapshots.SnapshotInfoTests.randomUserMetadata; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + public class CreateSnapshotRequestTests extends ESTestCase { // tests creating XContent and parsing with source(Map) equivalency @@ -80,6 +86,10 @@ public void testToXContent() throws IOException { original.includeGlobalState(randomBoolean()); } + if (randomBoolean()) { + original.userMetadata(randomUserMetadata()); + } + if (randomBoolean()) { Collection wildcardStates = randomSubsetOf(Arrays.asList(WildcardStates.values())); Collection