Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add versioning for UploadedIndexMetadata #14677

Merged
merged 2 commits into from
Jul 11, 2024
Merged

Add versioning for UploadedIndexMetadata #14677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5deb182
Make componentPrefix optional for backward compatibility
soosinha Jul 8, 2024
e4b5e70
Add versioning for UploadedIndexMetadata
soosinha Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 52 additions & 13 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.gateway.remote.ClusterMetadataManifest.Builder;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -243,7 +244,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_FIELD
);
parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
Expand Down Expand Up @@ -277,7 +278,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_ROUTING_FIELD
);
parser.declareNamedObject(
Expand Down Expand Up @@ -1112,16 +1113,30 @@ private static String componentPrefix(Object[] fields) {
return (String) fields[3];
}

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER = new ConstructingObjectParser<>(
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
declareParser(PARSER_V0, CODEC_V0);
declareParser(PARSER_V2, CODEC_V2);
}

private static void declareParser(ConstructingObjectParser<UploadedIndexMetadata, Void> parser, long codec_version) {
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
if (codec_version >= CODEC_V2) {
parser.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
}
}

static final String COMPONENT_PREFIX = "index--";
Expand All @@ -1130,15 +1145,32 @@ private static String componentPrefix(Object[] fields) {
private final String indexUUID;
private final String uploadedFilename;

private long codecVersion = CODEC_V2;

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX);
this(indexName, indexUUID, uploadedFileName, CODEC_V2);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, long codecVersion) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, codecVersion);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) {
this(indexName, indexUUID, uploadedFileName, componentPrefix, CODEC_V2);
}

public UploadedIndexMetadata(
String indexName,
String indexUUID,
String uploadedFileName,
String componentPrefix,
long codecVersion
) {
this.componentPrefix = componentPrefix;
this.indexName = indexName;
this.indexUUID = indexUUID;
this.uploadedFilename = uploadedFileName;
this.codecVersion = codecVersion;
}

public UploadedIndexMetadata(StreamInput in) throws IOException {
Expand Down Expand Up @@ -1175,10 +1207,13 @@ public String getComponentPrefix() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
.field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID())
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath())
.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath());
if (codecVersion >= CODEC_V2) {
builder.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
}
return builder;
}

@Override
Expand Down Expand Up @@ -1214,9 +1249,13 @@ public String toString() {
return Strings.toString(MediaTypeRegistry.JSON, this);
}

public static UploadedIndexMetadata fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
public static UploadedIndexMetadata fromXContent(XContentParser parser, long codecVersion) throws IOException {
if (codecVersion >= CODEC_V2) {
return CURRENT_PARSER.parse(parser, null);
}
return PARSER_V0.parse(parser, null);
}

}

/**
Expand Down
23 changes: 21 additions & 2 deletions server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
public class ClusterMetadataManifestTests extends OpenSearchTestCase {

public void testClusterMetadataManifestXContentV0() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand All @@ -74,7 +74,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
}

public void testClusterMetadataManifestXContentV1() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V1);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand Down Expand Up @@ -619,6 +619,24 @@ public void testUploadedIndexMetadataSerializationEqualsHashCode() {
);
}

public void testUploadedIndexMetadataWithoutComponentPrefix() throws IOException {
final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadata(
"test-index",
"test-index-uuid",
"test_file_name",
CODEC_V1
);
final XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
originalUploadedIndexMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();

try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser, 1L);
assertEquals(originalUploadedIndexMetadata, fromXContentUploadedIndexMetadata);
}
}

private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedIndexMetadata uploadedIndexMetadata) {
switch (randomInt(2)) {
case 0:
Expand All @@ -642,4 +660,5 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde
}
return uploadedIndexMetadata;
}

}
14 changes: 8 additions & 6 deletions server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2254,13 +2254,14 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
.stateVersion(1L)
.stateUUID("state-uuid")
.clusterUUID("cluster-uuid")
.codecVersion(CODEC_V2)
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.previousClusterUUID("prev-cluster-uuid")
.build();

BlobContainer blobContainer = mockBlobStoreObjects();
mockBlobContainer(blobContainer, expectedManifest, Map.of());
mockBlobContainer(blobContainer, expectedManifest, Map.of(), CODEC_V2);
when(blobContainer.readBlob(uploadedIndexMetadata.getUploadedFilename())).thenThrow(FileNotFoundException.class);

remoteClusterStateService.start();
Expand Down Expand Up @@ -2288,11 +2289,11 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
.clusterUUID("cluster-uuid")
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.codecVersion(ClusterMetadataManifest.CODEC_V0)
.codecVersion(CODEC_V2)
.previousClusterUUID("prev-cluster-uuid")
.build();

mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>());
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2);
remoteClusterStateService.start();
final ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
clusterState.getClusterName().value(),
Expand Down Expand Up @@ -2416,10 +2417,10 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.previousClusterUUID("prev-cluster-uuid")
.codecVersion(ClusterMetadataManifest.CODEC_V0)
.codecVersion(CODEC_V2)
.build();

mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata), CODEC_V2);

Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestClusterState(
clusterState.getClusterName().value(),
Expand Down Expand Up @@ -2664,6 +2665,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx
.clusterUUID("cluster-uuid")
.previousClusterUUID("prev-cluster-uuid")
.routingTableVersion(1)
.codecVersion(CODEC_V2)
.indicesRouting(List.of(uploadedIndiceRoutingMetadata))
.build();

Expand Down Expand Up @@ -3081,7 +3083,7 @@ private void mockBlobContainerForGlobalMetadata(
FORMAT_PARAMS
);
when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
if (codecVersion >= ClusterMetadataManifest.CODEC_V2) {
if (codecVersion >= CODEC_V2) {
String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
when(blobContainer.readBlob(COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer((invocationOnMock) -> {
BytesReference bytesReference = COORDINATION_METADATA_FORMAT.serialize(
Expand Down
Loading