Skip to content

Commit

Permalink
[Snapshot Interop] Add Changes in Restore Snapshot Flow for remote st…
Browse files Browse the repository at this point in the history 
…ore Interoperability (#6788)

* [Snapshot Interop] Add Changes in Restore Snapshot Flow for remote store interoperability.
---------

Signed-off-by: Harish Bhakuni <hbhakuni@amazon.com>
Co-authored-by: Harish Bhakuni <hbhakuni@amazon.com>
  • Loading branch information
@harishbhakuni
harishbhakuni and Harish Bhakuni authored Jun 27, 2023
1 parent f936b89 commit 317dd03
Show file tree
Hide file tree
Showing 28 changed files with 1,303 additions and 87 deletions.
4 changes: 4 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@
"type":"boolean",
"description":"Should this request wait until the operation has completed before returning",
"default":false
},
"source_remote_store_repository": {
"type":"string",
"description":"Remote Store Repository of Remote Store Indices"
}
},
"body":{
Expand Down
513 changes: 513 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
View file

Large diffs are not rendered by default.

42 changes: 40 additions & 2 deletions ...in/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ private static StorageType fromString(String string) {
private Settings indexSettings = EMPTY_SETTINGS;
private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
private StorageType storageType = StorageType.LOCAL;
@Nullable
private String sourceRemoteStoreRepository = null;

@Nullable // if any snapshot UUID will do
private String snapshotUuid;
Expand Down Expand Up @@ -148,6 +150,9 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_2_7_0)) {
storageType = in.readEnum(StorageType.class);
}
if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
sourceRemoteStoreRepository = in.readOptionalString();
}
}

@Override
Expand All @@ -169,6 +174,9 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
out.writeEnum(storageType);
}
if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
out.writeOptionalString(sourceRemoteStoreRepository);
}
}

@Override
Expand Down Expand Up @@ -521,6 +529,25 @@ public StorageType storageType() {
return storageType;
}

/**
* Sets Source Remote Store Repository for all the restored indices
*
* @param sourceRemoteStoreRepository name of the remote store repository that should be used for all restored indices.
*/
public RestoreSnapshotRequest setSourceRemoteStoreRepository(String sourceRemoteStoreRepository) {
this.sourceRemoteStoreRepository = sourceRemoteStoreRepository;
return this;
}

/**
* Returns Source Remote Store Repository for all the restored indices
*
* @return source Remote Store Repository
*/
public String getSourceRemoteStoreRepository() {
return sourceRemoteStoreRepository;
}

/**
* Parses restore definition
*
Expand Down Expand Up @@ -586,6 +613,12 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
throw new IllegalArgumentException("malformed storage_type");
}

} else if (name.equals("source_remote_store_repository")) {
if (entry.getValue() instanceof String) {
setSourceRemoteStoreRepository((String) entry.getValue());
} else {
throw new IllegalArgumentException("malformed source_remote_store_repository");
}
} else {
if (IndicesOptions.isIndicesOptions(name) == false) {
throw new IllegalArgumentException("Unknown parameter " + name);
Expand Down Expand Up @@ -631,6 +664,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (storageType != null) {
storageType.toXContent(builder);
}
if (sourceRemoteStoreRepository != null) {
builder.field("source_remote_store_repository", sourceRemoteStoreRepository);
}
builder.endObject();
return builder;
}
Expand Down Expand Up @@ -658,7 +694,8 @@ public boolean equals(Object o) {
&& Objects.equals(indexSettings, that.indexSettings)
&& Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings)
&& Objects.equals(snapshotUuid, that.snapshotUuid)
&& Objects.equals(storageType, that.storageType);
&& Objects.equals(storageType, that.storageType)
&& Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository);
}

@Override
Expand All @@ -675,7 +712,8 @@ public int hashCode() {
includeAliases,
indexSettings,
snapshotUuid,
storageType
storageType,
sourceRemoteStoreRepository
);
result = 31 * result + Arrays.hashCode(indices);
result = 31 * result + Arrays.hashCode(ignoreIndexSettings);
Expand Down
8 changes: 8 additions & 0 deletions .../org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,4 +256,12 @@ public RestoreSnapshotRequestBuilder setStorageType(RestoreSnapshotRequest.Stora
request.storageType(storageType);
return this;
}

/**
* Sets the source remote store repository name
*/
public RestoreSnapshotRequestBuilder setSourceRemoteStoreRepository(String repositoryName) {
request.setSourceRemoteStoreRepository(repositoryName);
return this;
}
}
50 changes: 45 additions & 5 deletions server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import org.opensearch.Version;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.Nullable;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
Expand Down Expand Up @@ -257,23 +258,29 @@ public static class SnapshotRecoverySource extends RecoverySource {
private final IndexId index;
private final Version version;
private final boolean isSearchableSnapshot;
private final boolean remoteStoreIndexShallowCopy;
private final String sourceRemoteStoreRepository;

public SnapshotRecoverySource(String restoreUUID, Snapshot snapshot, Version version, IndexId indexId) {
this(restoreUUID, snapshot, version, indexId, false);
this(restoreUUID, snapshot, version, indexId, false, false, null);
}

public SnapshotRecoverySource(
String restoreUUID,
Snapshot snapshot,
Version version,
IndexId indexId,
boolean isSearchableSnapshot
boolean isSearchableSnapshot,
boolean remoteStoreIndexShallowCopy,
@Nullable String sourceRemoteStoreRepository
) {
this.restoreUUID = restoreUUID;
this.snapshot = Objects.requireNonNull(snapshot);
this.version = Objects.requireNonNull(version);
this.index = Objects.requireNonNull(indexId);
this.isSearchableSnapshot = isSearchableSnapshot;
this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy;
this.sourceRemoteStoreRepository = sourceRemoteStoreRepository;
}

SnapshotRecoverySource(StreamInput in) throws IOException {
Expand All @@ -286,6 +293,13 @@ public SnapshotRecoverySource(
} else {
isSearchableSnapshot = false;
}
if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
remoteStoreIndexShallowCopy = in.readBoolean();
sourceRemoteStoreRepository = in.readOptionalString();
} else {
remoteStoreIndexShallowCopy = false;
sourceRemoteStoreRepository = null;
}
}

public String restoreUUID() {
Expand Down Expand Up @@ -314,6 +328,14 @@ public boolean isSearchableSnapshot() {
return isSearchableSnapshot;
}

public String sourceRemoteStoreRepository() {
return sourceRemoteStoreRepository;
}

public boolean remoteStoreIndexShallowCopy() {
return remoteStoreIndexShallowCopy;
}

@Override
protected void writeAdditionalFields(StreamOutput out) throws IOException {
out.writeString(restoreUUID);
Expand All @@ -323,6 +345,10 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
out.writeBoolean(isSearchableSnapshot);
}
if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
out.writeBoolean(remoteStoreIndexShallowCopy);
out.writeOptionalString(sourceRemoteStoreRepository);
}
}

@Override
Expand All @@ -337,7 +363,9 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param
.field("version", version.toString())
.field("index", index.getName())
.field("restoreUUID", restoreUUID)
.field("isSearchableSnapshot", isSearchableSnapshot);
.field("isSearchableSnapshot", isSearchableSnapshot)
.field("remoteStoreIndexShallowCopy", remoteStoreIndexShallowCopy)
.field("sourceRemoteStoreRepository", sourceRemoteStoreRepository);
}

@Override
Expand All @@ -359,12 +387,24 @@ public boolean equals(Object o) {
&& snapshot.equals(that.snapshot)
&& index.equals(that.index)
&& version.equals(that.version)
&& isSearchableSnapshot == that.isSearchableSnapshot;
&& isSearchableSnapshot == that.isSearchableSnapshot
&& remoteStoreIndexShallowCopy == that.remoteStoreIndexShallowCopy
&& sourceRemoteStoreRepository != null
? sourceRemoteStoreRepository.equals(that.sourceRemoteStoreRepository)
: that.sourceRemoteStoreRepository == null;
}

@Override
public int hashCode() {
return Objects.hash(restoreUUID, snapshot, index, version, isSearchableSnapshot);
return Objects.hash(
restoreUUID,
snapshot,
index,
version,
isSearchableSnapshot,
remoteStoreIndexShallowCopy,
sourceRemoteStoreRepository
);
}

}
Expand Down
Loading

0 comments on commit 317dd03

Please sign in to comment.