diff --git a/ambry-account/src/integration-test/java/com/github/ambry/account/MySqlAccountServiceIntegrationTest.java b/ambry-account/src/integration-test/java/com/github/ambry/account/MySqlAccountServiceIntegrationTest.java index 4220413391..3a785ba245 100644 --- a/ambry-account/src/integration-test/java/com/github/ambry/account/MySqlAccountServiceIntegrationTest.java +++ b/ambry-account/src/integration-test/java/com/github/ambry/account/MySqlAccountServiceIntegrationTest.java @@ -39,11 +39,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.json.JSONArray; import org.json.JSONObject; @@ -72,6 +73,8 @@ public class MySqlAccountServiceIntegrationTest { private MySqlAccountServiceConfig accountServiceConfig; private MySqlAccountService mySqlAccountService; private static final String DATASET_NAME = "testDataset"; + private static final String DATASET_NAME_RENAME = "testDatasetRename"; + private static final String DATASET_NAME_BASIC = "testDatasetBasic"; private static final String DATASET_NAME_NOT_EXIST = "testDatasetNotExist"; private static final String DATASET_NAME_WITH_TTL = "testDatasetWithTtl"; @@ -728,6 +731,236 @@ public void testConnectionRefreshOnException() throws Exception { assertEquals("Mismatch in account read from db", a1, mySqlAccountStore.getNewAccounts(0).iterator().next()); } + @Test + public void testRenameDatasetVersion() throws Exception { + Account testAccount = makeTestAccountWithContainer(); + Container testContainer = new ArrayList<>(testAccount.getAllContainers()).get(0); + long ttl = 36000L; + Dataset dataset = + new DatasetBuilder(testAccount.getName(), testContainer.getName(), DATASET_NAME_RENAME).setVersionSchema( + Dataset.VersionSchema.SEMANTIC_LONG).setRetentionCount(1).setRetentionTimeInSeconds(ttl).build(); + Set expectedFinalVersion = new HashSet<>(); + Set allRenamedVersionsInDb = new HashSet<>(); + // Add a dataset to db + mySqlAccountStore.addDataset(testAccount.getId(), testContainer.getId(), dataset); + // add source dataset version + String sourceVersion = "999.999.999.999"; + allRenamedVersionsInDb.add(sourceVersion); + DatasetVersionRecord expectedDatasetVersionRecord = + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_RENAME, sourceVersion, -1, + null); + long creationTime = System.currentTimeMillis(); + mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, -1, creationTime, false, + DatasetVersionState.READY); + // rename to a target dataset version + // 999.999.999.999 -> 1.1.1.1(succeed) + String targetVersion = "1.1.1.1"; + expectedFinalVersion.add(targetVersion); + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + // get the renamed version. + DatasetVersionRecord renamedDatasetVersion = + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, targetVersion); + //verify each field + assertEquals("Mismatch in dataset version's accountId", expectedDatasetVersionRecord.getAccountId(), + renamedDatasetVersion.getAccountId()); + assertEquals("Mismatch in dataset version's containerId", expectedDatasetVersionRecord.getContainerId(), + renamedDatasetVersion.getContainerId()); + assertEquals("Mismatch in dataset version's datasetName", expectedDatasetVersionRecord.getDatasetName(), + renamedDatasetVersion.getDatasetName()); + assertEquals("Mismatch in dataset version's ", creationTime + dataset.getRetentionTimeInSeconds() * 1000, + renamedDatasetVersion.getExpirationTimeMs()); + assertEquals("Mismatch in dataset version's ", targetVersion, renamedDatasetVersion.getVersion()); + + //get previous version and should be deleted. + try { + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + + // sourceVersion is auto incr version, should fail. + sourceVersion = "MAJOR"; + try { + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + fail(); + } catch (IllegalArgumentException e) { + //no-op + } + + // targetVersion is auto incr version, should fail. + sourceVersion = "0.0.0.0"; + targetVersion = "MAJOR"; + + try { + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + fail(); + } catch (IllegalArgumentException e) { + //no-op + } + + // rename a version which does not exist, should fail + // 2.2.2.2 -> 888.888.888.888(fail) + sourceVersion = "2.2.2.2"; + targetVersion = "888.888.888.888"; + + try { + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + + // rename to an existing version, should throw conflict. + // rename a version which does not exist, should fail + // 1.1.1.1 -> 1.1.1.1(fail) + sourceVersion = "1.1.1.1"; + targetVersion = "1.1.1.1"; + + try { + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.ResourceConflict, e.getErrorCode()); + } + + //Add a renamed version, should fail with conflict + String renamedVersion = "1.1.1.1"; + try { + mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, renamedVersion, -1, System.currentTimeMillis(), false, + DatasetVersionState.READY); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.ResourceConflict, e.getErrorCode()); + } + + //rename a renamed version, should fail with not found + //1.1.1.1 -> 888.888.888.888(fail) + sourceVersion = "999.999.999.999"; + targetVersion = "888.888.888.888"; + try { + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion, targetVersion); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + + //rename twice, and it should point to the original version. + //998.998.998.998 -> 997.997.997.997(succeed) + String originalSourceVersion = "998.998.998.998"; + allRenamedVersionsInDb.add(originalSourceVersion); + mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, originalSourceVersion, -1, System.currentTimeMillis(), false, + DatasetVersionState.READY); + + targetVersion = "997.997.997.997"; + allRenamedVersionsInDb.add(targetVersion); + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, originalSourceVersion, targetVersion); + DatasetVersionRecord renamedDatasetVersionFirstTime = + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, targetVersion); + assertEquals("Rename from does not match", originalSourceVersion, renamedDatasetVersionFirstTime.getRenameFrom()); + + //997.997.997.997 -> 3.3.3.3(succeed) + String renamedSourceVersion = "997.997.997.997"; + targetVersion = "3.3.3.3"; + expectedFinalVersion.add(targetVersion); + mySqlAccountStore.renameDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, renamedSourceVersion, targetVersion); + DatasetVersionRecord renamedDatasetVersionSecondTime = + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, targetVersion); + assertEquals("Rename from does not match", originalSourceVersion, renamedDatasetVersionSecondTime.getRenameFrom()); + + //list the dataset version. + Page allDatasetVersions = + mySqlAccountStore.listAllValidDatasetVersions(testAccount.getId(), testContainer.getId(), DATASET_NAME_RENAME, + null); + Set allVersions = new HashSet<>(allDatasetVersions.getEntries()); + assertEquals("Final versions should match", allVersions, expectedFinalVersion); + + //get Valid version out of retention + //at this moment, we have 1.1.1.1 and 3.3.3.3 which is the valid version + //999.999.999.999, 998.998.998.998, 997.997.997.997 has been renamed. + List datasetVersionOutOfRetention = + mySqlAccountStore.getAllValidVersionsOutOfRetentionCount(testAccount.getId(), testContainer.getId(), + testAccount.getName(), testContainer.getName(), DATASET_NAME_RENAME); + assertEquals("Unexpected out of retention count", 1, datasetVersionOutOfRetention.size()); + + //If I want to delete a dataset with all versions, I should include the renamed state dataset version. + List datasetVersionRecordsForDatasetDeletion = + mySqlAccountStore.getAllValidVersionForDatasetDeletion(testAccount.getId(), testContainer.getId(), + DATASET_NAME_RENAME); + assertEquals("Unexpected delete numbers", 5, datasetVersionRecordsForDatasetDeletion.size()); + + //Update dataset version + for (String version : allRenamedVersionsInDb) { + try { + mySqlAccountStore.updateDatasetVersionTtl(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, version); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + } + + for (String version : expectedFinalVersion) { + mySqlAccountStore.updateDatasetVersionTtl(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, version); + DatasetVersionRecord datasetVersionRecordFromDb = + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, version); + assertEquals("The dataset version should be permanent after ttl update.", -1, + datasetVersionRecordFromDb.getExpirationTimeMs()); + mySqlAccountStore.deleteDatasetVersion(testAccount.getId(), testContainer.getId(), DATASET_NAME_RENAME, version); + } + + //DELETE a version which been renamed, should not be found? + for (String version : allRenamedVersionsInDb) { + try { + mySqlAccountStore.deleteDatasetVersion(testAccount.getId(), testContainer.getId(), DATASET_NAME_RENAME, + version); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + } + + for (String version : allRenamedVersionsInDb) { + //If the delete is to delete the dataset, + mySqlAccountStore.deleteDatasetVersionForDatasetDelete(testAccount.getId(), testContainer.getId(), + DATASET_NAME_RENAME, version); + try { + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, version); + fail(); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.NotFound, e.getErrorCode()); + } + } + + //add a deleted version + sourceVersion = "1.1.1.1"; + mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, renamedVersion, -1, System.currentTimeMillis(), false, + DatasetVersionState.READY); + DatasetVersionRecord datasetVersionRecord = + mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), + testContainer.getName(), DATASET_NAME_RENAME, sourceVersion); + assertNull("Rename from should be null", datasetVersionRecord.getRenameFrom()); + } + @Test public void testDatasetVersionState() throws Exception { Account testAccount = makeTestAccountWithContainer(); @@ -742,7 +975,7 @@ public void testDatasetVersionState() throws Exception { DatasetVersionRecord datasetVersionRecordFromMysql; String version = "LATEST"; DatasetVersionRecord expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "1", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "1", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, System.currentTimeMillis(), false, @@ -760,7 +993,7 @@ public void testDatasetVersionState() throws Exception { //Upload a new latest version, should be version 2 since we have version 1 in progress expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, System.currentTimeMillis(), false, @@ -979,7 +1212,7 @@ public void testLatestVersionSupport() throws Exception { // Add a LATEST dataset version DatasetVersionRecord expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "1", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "1", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, System.currentTimeMillis(), false, @@ -988,7 +1221,7 @@ public void testLatestVersionSupport() throws Exception { // Add a new LATEST dataset version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, System.currentTimeMillis(), false, @@ -997,7 +1230,7 @@ public void testLatestVersionSupport() throws Exception { // Get the LATEST dataset version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version); @@ -1006,7 +1239,7 @@ public void testLatestVersionSupport() throws Exception { // Deleted version 2 and add new latest version mySqlAccountStore.deleteDatasetVersion(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2"); expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, "2", -1, null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, System.currentTimeMillis(), false, @@ -1034,7 +1267,8 @@ public void testLatestVersionSupport() throws Exception { // add a major version. version = "MAJOR"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "1.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "1.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1042,7 +1276,8 @@ public void testLatestVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second major version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1052,7 +1287,8 @@ public void testLatestVersionSupport() throws Exception { // add a patch version. version = "PATCH"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.1", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.1", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1060,7 +1296,8 @@ public void testLatestVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second path version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.0.2", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1070,7 +1307,8 @@ public void testLatestVersionSupport() throws Exception { // add a minor version. version = "MINOR"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.1.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.1.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1078,7 +1316,8 @@ public void testLatestVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second minor version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.2.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.2.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version, -1, System.currentTimeMillis(), false, @@ -1088,7 +1327,8 @@ public void testLatestVersionSupport() throws Exception { //get the latest version. version = "LATEST"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.2.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC, "2.2.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC, version); @@ -1153,7 +1393,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { // add a major version. version = "MAJOR"; DatasetVersionRecord expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "1.0.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "1.0.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1161,7 +1402,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second major version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1171,7 +1413,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { // add a revision version. version = "REVISION"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.1", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.1", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1179,7 +1422,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second revision version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.2", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.0.2", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1189,7 +1433,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { // add a patch version. version = "PATCH"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.1.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.1.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1197,7 +1442,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second path version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.2.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.0.2.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1207,7 +1453,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { // add a minor version. version = "MINOR"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.1.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.1.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1215,7 +1462,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { assertEquals("Mismatch in dataset", expectedDatasetVersionRecord, datasetVersionRecordFromMysql); // add second minor version expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.2.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.2.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version, -1, System.currentTimeMillis(), false, @@ -1225,7 +1473,8 @@ public void testLatestSemanticLongVersionSupport() throws Exception { //get the latest version. version = "LATEST"; expectedDatasetVersionRecord = - new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.2.0.0", -1); + new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME_WITH_SEMANTIC_LONG, "2.2.0.0", -1, + null); datasetVersionRecordFromMysql = mySqlAccountStore.getDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME_WITH_SEMANTIC_LONG, version); @@ -1288,7 +1537,7 @@ public void testAddAndGetDatasetVersion() throws Exception { long creationTimeInMs = System.currentTimeMillis(); DatasetVersionRecord expectedDatasetVersionRecord = new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, version, - Utils.addSecondsToEpochTime(creationTimeInMs, datasetTtl)); + Utils.addSecondsToEpochTime(creationTimeInMs, datasetTtl), null); DatasetVersionRecord datasetVersionRecordFromMysql = mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, -1, creationTimeInMs, false, DatasetVersionState.READY); @@ -1311,7 +1560,7 @@ public void testAddAndGetDatasetVersion() throws Exception { boolean datasetVersionTtlEnable = true; expectedDatasetVersionRecord = new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, version, - Utils.addSecondsToEpochTime(creationTimeInMs, datasetVersionTtl)); + Utils.addSecondsToEpochTime(creationTimeInMs, datasetVersionTtl), null); mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, datasetVersionTtl, creationTimeInMs, datasetVersionTtlEnable, DatasetVersionState.READY); @@ -1329,7 +1578,7 @@ public void testAddAndGetDatasetVersion() throws Exception { versions.add(versionNumber); expectedDatasetVersionRecord = new DatasetVersionRecord(testAccount.getId(), testContainer.getId(), DATASET_NAME, version, - Utils.addSecondsToEpochTime(creationTimeInMs, datasetTtl)); + Utils.addSecondsToEpochTime(creationTimeInMs, datasetTtl), null); mySqlAccountStore.addDatasetVersion(testAccount.getId(), testContainer.getId(), testAccount.getName(), testContainer.getName(), DATASET_NAME, version, datasetVersionTtl, creationTimeInMs, datasetVersionTtlEnable, DatasetVersionState.READY); diff --git a/ambry-account/src/main/java/com/github/ambry/account/CachedAccountService.java b/ambry-account/src/main/java/com/github/ambry/account/CachedAccountService.java index ae0bb63709..a2336e1124 100644 --- a/ambry-account/src/main/java/com/github/ambry/account/CachedAccountService.java +++ b/ambry-account/src/main/java/com/github/ambry/account/CachedAccountService.java @@ -621,6 +621,36 @@ public void deleteDatasetVersion(String accountName, String containerName, Strin } } + @Override + public void renameDatasetVersion(String accountName, String containerName, String datasetName, String sourceVersion, + String targetVersion) throws AccountServiceException { + try { + Pair accountAndContainerIdPair = getAccountAndContainerIdFromName(accountName, containerName); + if (mySqlAccountStore == null) { + mySqlAccountStore = this.supplier.get(); + } + mySqlAccountStore.renameDatasetVersion(accountAndContainerIdPair.getFirst(), + accountAndContainerIdPair.getSecond(), accountName, containerName, datasetName, sourceVersion, targetVersion); + } catch (SQLException e) { + throw translateSQLException(e); + } + } + + @Override + public void deleteDatasetVersionForDatasetDelete(String accountName, String containerName, String datasetName, + String version) throws AccountServiceException { + try { + Pair accountAndContainerIdPair = getAccountAndContainerIdFromName(accountName, containerName); + if (mySqlAccountStore == null) { + mySqlAccountStore = this.supplier.get(); + } + mySqlAccountStore.deleteDatasetVersionForDatasetDelete(accountAndContainerIdPair.getFirst(), + accountAndContainerIdPair.getSecond(), datasetName, version); + } catch (SQLException e) { + throw translateSQLException(e); + } + } + @Override public void updateDatasetVersionTtl(String accountName, String containerName, String datasetName, String version) throws AccountServiceException { diff --git a/ambry-account/src/main/java/com/github/ambry/account/MySqlAccountService.java b/ambry-account/src/main/java/com/github/ambry/account/MySqlAccountService.java index 201cf63867..a6ef9f23b5 100644 --- a/ambry-account/src/main/java/com/github/ambry/account/MySqlAccountService.java +++ b/ambry-account/src/main/java/com/github/ambry/account/MySqlAccountService.java @@ -254,12 +254,24 @@ public DatasetVersionRecord getDatasetVersion(String accountName, String contain return cachedAccountService.getDatasetVersion(accountName, containerName, datasetName, version); } + @Override + public void renameDatasetVersion(String accountName, String containerName, String datasetName, String sourceVersion, + String targetVersion) throws AccountServiceException { + cachedAccountService.renameDatasetVersion(accountName, containerName, datasetName, sourceVersion, targetVersion); + } + @Override public void deleteDatasetVersion(String accountName, String containerName, String datasetName, String version) throws AccountServiceException { cachedAccountService.deleteDatasetVersion(accountName, containerName, datasetName, version); } + @Override + public void deleteDatasetVersionForDatasetDelete(String accountName, String containerName, String datasetName, + String version) throws AccountServiceException { + cachedAccountService.deleteDatasetVersionForDatasetDelete(accountName, containerName, datasetName, version); + } + @Override public void updateDatasetVersionTtl(String accountName, String containerName, String datasetName, String version) throws AccountServiceException { diff --git a/ambry-account/src/main/java/com/github/ambry/account/mysql/DatasetDao.java b/ambry-account/src/main/java/com/github/ambry/account/mysql/DatasetDao.java index f321da356a..ed19d0ccbf 100644 --- a/ambry-account/src/main/java/com/github/ambry/account/mysql/DatasetDao.java +++ b/ambry-account/src/main/java/com/github/ambry/account/mysql/DatasetDao.java @@ -30,11 +30,15 @@ import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import com.github.ambry.account.DatasetVersionRecord; import com.github.ambry.protocol.DatasetVersionState; +import java.util.Set; +import static com.github.ambry.account.Dataset.VersionSchema.*; import static com.github.ambry.mysql.MySqlDataAccessor.OperationType.*; import static com.github.ambry.utils.Utils.*; @@ -68,6 +72,7 @@ public class DatasetDao { public static final String RETENTION_TIME_IN_SECONDS = "retentionTimeInSeconds"; public static final String USER_TAGS = "userTags"; public static final String DELETED_TS = "deleted_ts"; + public static final String RENAME_FROM = "rename_from"; // Dataset version table fields public static final String DATASET_VERSION_TABLE = "DatasetVersions"; @@ -81,8 +86,10 @@ public class DatasetDao { // Dataset version table query strings. private final String insertDatasetVersionSql; private final String updateDatasetVersionStateSql; + private final String updateDatasetVersionStateAndDeletedTsSql; private final String getDatasetVersionByNameSql; private final String deleteDatasetVersionByIdSql; + private final String deleteDatasetVersionForDatasetDeleteByIdSql; private final String updateDatasetVersionIfExpiredSql; private final String listLatestVersionSqlForUpload; private final String getLatestVersionSqlForDownload; @@ -91,6 +98,7 @@ public class DatasetDao { private final String listValidDatasetVersionsByPageSql; private final String listValidDatasetVersionsByListSql; private final String updateDatasetVersionTtlSql; + private final String copyToNewDatasetVersionSql; // Dataset table query strings private final String insertDatasetSql; @@ -100,6 +108,7 @@ public class DatasetDao { private final String updateDatasetIfExpiredSql; private final String deleteDatasetByIdSql; private final String listValidDatasetsSql; + private final String getDatasetVersionRenameFromSql; public DatasetDao(MySqlDataAccessor dataAccessor, MySqlAccountServiceConfig mySqlAccountServiceConfig, MySqlMetrics metrics) { @@ -146,6 +155,13 @@ public DatasetDao(MySqlDataAccessor dataAccessor, MySqlAccountServiceConfig mySq String.format("insert into %s (%s, %s, %s, %s, %s, %s, %s, %s) values (?, ?, ?, ?, now(3), now(3), ?, ?)", DATASET_VERSION_TABLE, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION, CREATION_TIME, LAST_MODIFIED_TIME, DELETED_TS, DATASET_VERSION_STATE); + //copy a dataset version from a source version. + //We need to update the modify time so counter based purge policy won't delete it. + copyToNewDatasetVersionSql = String.format( + "INSERT INTO %1$s (%2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s) SELECT %2$s, %3$s, %4$s, ?, " + + "%6$s, NOW(3), %8$s, %9$s, ? FROM %1$s WHERE %2$s = ? AND %3$s = ? AND %4$s = ? AND %5$s = ? AND %9$s = ?", + DATASET_VERSION_TABLE, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION, CREATION_TIME, LAST_MODIFIED_TIME, + DELETED_TS, DATASET_VERSION_STATE, RENAME_FROM); //dataset version has in_progress and ready states. //when we put the dataset version, the flow is add dataset version in_progress -> add named blob -> add regular blob -> update dataset version to ready state. //Only the dataset version in ready state will be considered as a valid version. @@ -153,23 +169,27 @@ public DatasetDao(MySqlDataAccessor dataAccessor, MySqlAccountServiceConfig mySq String.format("update %s set %s = ?, %s = now(3) where %s = ? and %s = ? and %s = ? and %s = ?", DATASET_VERSION_TABLE, DATASET_VERSION_STATE, LAST_MODIFIED_TIME, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION); + updateDatasetVersionStateAndDeletedTsSql = + String.format("update %s set %s = ?, %s = now(3), %s = NULL where %s = ? and %s = ? and %s = ? and %s = ?", + DATASET_VERSION_TABLE, DATASET_VERSION_STATE, LAST_MODIFIED_TIME, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, + DATASET_NAME, VERSION); //get the current latest version to download when user provide version == LATEST/MAJOR/MINOR/PATCH - getLatestVersionSqlForDownload = String.format("select %1$s, %2$s from %3$s " + getLatestVersionSqlForDownload = String.format("select %1$s, %2$s, %9$s from %3$s " + "where (%4$s IS NULL or %4$s > now(3)) and (%5$s, %6$s, %7$s, %8$s) = (?, ?, ?, ?) ORDER BY %1$s DESC LIMIT 1", VERSION, DELETED_TS, DATASET_VERSION_TABLE, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, - DATASET_VERSION_STATE); + DATASET_VERSION_STATE, RENAME_FROM); //get the latest version + 1 for upload when user provide version == LATEST/MAJOR/MINOR/PATCH listLatestVersionSqlForUpload = String.format("select %1$s from %2$s " + "where (%3$s IS NULL or %3$s > now(3)) and (%4$s, %5$s, %6$s) = (?, ?, ?) ORDER BY %1$s DESC LIMIT 1", VERSION, DATASET_VERSION_TABLE, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME); //when we delete a dataset, we will delete all the versions under the dataset. This is to list all versions under a dataset. - listValidVersionForDatasetDeletionSql = - String.format("select %s, %s from %s " + "where (%s IS NULL or %s > now(3)) and %s = ? and %s = ? and %s = ?", - VERSION, DELETED_TS, DATASET_VERSION_TABLE, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME); + listValidVersionForDatasetDeletionSql = String.format( + "select %s, %s, %s from %s where (%s IS NULL or %s > now(3)) and %s = ? and %s = ? and %s = ?", VERSION, + RENAME_FROM, DELETED_TS, DATASET_VERSION_TABLE, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME); // list all valid versions sorted by last modified time, and skip the first N records which is not out of retentionCount. - listVersionByModifiedTimeAndFilterByRetentionSql = String.format("select %s, %s from %s " + listVersionByModifiedTimeAndFilterByRetentionSql = String.format("select %s, %s, %s from %s " + "where (%s IS NULL or %s > now(3)) and %s = ? and %s = ? and %s = ? and %s = ? ORDER BY %s DESC LIMIT ?, 100", - VERSION, DELETED_TS, DATASET_VERSION_TABLE, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, + VERSION, RENAME_FROM, DELETED_TS, DATASET_VERSION_TABLE, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, DATASET_VERSION_STATE, LAST_MODIFIED_TIME); //list dataset versions under a dataset by page. listValidDatasetVersionsByPageSql = String.format("select %1$s from %2$s " @@ -177,28 +197,35 @@ public DatasetDao(MySqlDataAccessor dataAccessor, MySqlAccountServiceConfig mySq + "ORDER BY %1$s ASC LIMIT ?", VERSION, DATASET_VERSION_TABLE, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, DATASET_VERSION_STATE); //this is used for customized retention policy where we need to provide all versions under a dataset ordered by version number. - listValidDatasetVersionsByListSql = String.format("select %1$s, %3$s, %8$s from %2$s " + listValidDatasetVersionsByListSql = String.format("select %1$s, %3$s, %8$s, %9$s from %2$s " + "where (%3$s IS NULL or %3$s > now(3)) and %4$s = ? and %5$s = ? and %6$s = ? and %7$s = ? " + "ORDER BY %1$s ASC", VERSION, DATASET_VERSION_TABLE, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, - DATASET_VERSION_STATE, CREATION_TIME); + DATASET_VERSION_STATE, CREATION_TIME, RENAME_FROM); //this is to update the dataset version to permanent. updateDatasetVersionTtlSql = String.format( "update %s set %s = NULL where %s = ? and %s = ? and %s = ? and %s = ? and (deleted_ts is NULL or deleted_ts > now(3)) and %s = ?", DATASET_VERSION_TABLE, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION, DATASET_VERSION_STATE); //get the dataset version. getDatasetVersionByNameSql = - String.format("select %s, %s from %s where %s = ? and %s = ? and %s = ? and %s = ? and %s = ?", LAST_MODIFIED_TIME, - DELETED_TS, DATASET_VERSION_TABLE, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION, DATASET_VERSION_STATE); + String.format("select %s, %s, %s from %s where %s = ? and %s = ? and %s = ? and %s = ? and %s = ?", LAST_MODIFIED_TIME, + DELETED_TS, RENAME_FROM, DATASET_VERSION_TABLE, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION, DATASET_VERSION_STATE); //delete a dataset version deleteDatasetVersionByIdSql = String.format( + "update %s set %s = now(3), %s = now(3) where (%s IS NULL or %s > now(3)) and %s = ? and %s = ? and %s = ? and %s = ? and %s <> ?", + DATASET_VERSION_TABLE, DELETED_TS, LAST_MODIFIED_TIME, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, + DATASET_NAME, VERSION, DATASET_VERSION_STATE); + deleteDatasetVersionForDatasetDeleteByIdSql = String.format( "update %s set %s = now(3), %s = now(3) where (%s IS NULL or %s > now(3)) and %s = ? and %s = ? and %s = ? and %s = ?", DATASET_VERSION_TABLE, DELETED_TS, LAST_MODIFIED_TIME, DELETED_TS, DELETED_TS, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION); //if dataset version is expired, we can create a dataset version with same primary key, but we have to use update instead of insert. updateDatasetVersionIfExpiredSql = String.format( - "update %s set %s = ?, %s = now(3), %s = now(3), %s = ?, %s = ? where %s = ? and %s = ? and %s = ? and %s = ? and deleted_ts < now(3)", - DATASET_VERSION_TABLE, VERSION, CREATION_TIME, LAST_MODIFIED_TIME, DELETED_TS, DATASET_VERSION_STATE, ACCOUNT_ID, - CONTAINER_ID, DATASET_NAME, VERSION); + "update %s set %s = ?, %s = now(3), %s = now(3), %s = ?, %s = ?, %s = NULL where %s = ? and %s = ? and %s = ? and %s = ? and deleted_ts < now(3)", + DATASET_VERSION_TABLE, VERSION, CREATION_TIME, LAST_MODIFIED_TIME, DELETED_TS, DATASET_VERSION_STATE, + RENAME_FROM, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION); + getDatasetVersionRenameFromSql = + String.format("SELECT %1$s FROM %2$s WHERE %3$s = ? AND %4$s = ? AND %5$s = ? AND %6$s = ?", RENAME_FROM, + DATASET_VERSION_TABLE, ACCOUNT_ID, CONTAINER_ID, DATASET_NAME, VERSION); } /** @@ -342,6 +369,37 @@ public DatasetVersionRecord addDatasetVersions(int accountId, int containerId, S } } + /** + * Rename a dataset version of {@link Dataset} + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param accountName the name for the parent account. + * @param containerName the name for the container. + * @param datasetName the name of the dataset. + * @param sourceVersion the source version to rename. + * @param targetVersion the target version to rename. + * @throws SQLException + * @throws AccountServiceException + */ + public void renameDatasetVersion(int accountId, int containerId, String accountName, String containerName, + String datasetName, String sourceVersion, String targetVersion) throws SQLException, AccountServiceException { + long startTimeMs = System.currentTimeMillis(); + try { + Dataset dataset = getDatasetHelper(accountId, containerId, accountName, containerName, datasetName, true); + if (!SEMANTIC_LONG.equals(dataset.getVersionSchema())) { + throw new IllegalArgumentException("Rename API only supported for SEMANTIC_LONG schema"); + } + if (isAutoIncrVersion(sourceVersion) || isAutoIncrVersion(targetVersion)) { + throw new IllegalArgumentException("Rename API can't rename from/to auto incr version"); + } + renameDatasetVersionHelper(accountId, containerId, datasetName, dataset.getVersionSchema(), sourceVersion, + targetVersion); + dataAccessor.onSuccess(Write, System.currentTimeMillis() - startTimeMs); + } catch (SQLException | AccountServiceException e) { + dataAccessor.onException(e, Write); + throw e; + } + } /** * Helper function to support add dataset verison. @@ -365,7 +423,91 @@ private DatasetVersionRecord addDatasetVersionHelper(int accountId, int containe long newExpirationTimeMs = executeAddDatasetVersionStatement(insertDatasetVersionStatement, accountId, containerId, dataset.getDatasetName(), versionNumber, timeToLiveInSeconds, creationTimeInMs, dataset, datasetVersionTtlEnabled, datasetVersionState); - return new DatasetVersionRecord(accountId, containerId, dataset.getDatasetName(), version, newExpirationTimeMs); + return new DatasetVersionRecord(accountId, containerId, dataset.getDatasetName(), version, newExpirationTimeMs, + null); + } + + /** + * Helper function to rename a dataset version from source version to target version. + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param sourceVersion the source version to rename. + * @param targetVersion the target version to rename. + * @throws SQLException + * @throws AccountServiceException + */ + private void renameDatasetVersionHelper(int accountId, int containerId, String datasetName, + Dataset.VersionSchema versionSchema, String sourceVersion, String targetVersion) + throws SQLException, AccountServiceException { + long sourceVersionValue = getVersionBasedOnSchema(sourceVersion, versionSchema); + long targetVersionValue = getVersionBasedOnSchema(targetVersion, versionSchema); + String renameFromSourceVersion = + getDatasetVersionRenameFromHelper(accountId, containerId, datasetName, sourceVersionValue); + //sanity check for the renameFromSourceVersion + if (renameFromSourceVersion != null) { + getVersionBasedOnSchema(renameFromSourceVersion, versionSchema); + } + try { + // Disable auto commits + dataAccessor.setAutoCommit(false); + PreparedStatement copyToNewDatasetVersionStatement = + dataAccessor.getPreparedStatement(copyToNewDatasetVersionSql, true); + //copy the source version to the new target version. + //if source version has a rename field, set to the rename field + String renameFromForTarget = renameFromSourceVersion != null? renameFromSourceVersion : sourceVersion; + executeCopyToNewDatasetVersionStatement(copyToNewDatasetVersionStatement, accountId, containerId, datasetName, + sourceVersionValue, targetVersionValue, renameFromForTarget); + //mark the original dataset version state to renamed. + PreparedStatement updateDatasetVersionStateAndDeletedTsStatement = + dataAccessor.getPreparedStatement(updateDatasetVersionStateAndDeletedTsSql, true); + executeUpdateDatasetVersionStateAndDeletedStatement(updateDatasetVersionStateAndDeletedTsStatement, accountId, + containerId, datasetName, sourceVersionValue, DatasetVersionState.RENAMED); + dataAccessor.commit(); + } catch (SQLException | AccountServiceException e) { + dataAccessor.rollback(); + dataAccessor.onException(e, Write); + if (e instanceof SQLIntegrityConstraintViolationException) { + throw new AccountServiceException(e.getMessage(), AccountServiceErrorCode.ResourceConflict); + } + throw e; + } finally { + // Close the connection to ensure subsequent queries are made in a new transaction and return the latest data + dataAccessor.closeActiveConnection(); + } + } + + private String getDatasetVersionRenameFromHelper(int accountId, int containerId, String datasetName, + long sourceVersionValue) throws SQLException, AccountServiceException { + PreparedStatement getDatasetVersionRenameFromStatement = + dataAccessor.getPreparedStatement(getDatasetVersionRenameFromSql, true); + try { + return executeGetDatasetVersionRenameStatement(getDatasetVersionRenameFromStatement, accountId, containerId, + datasetName, sourceVersionValue); + } catch (AccountServiceException e) { + dataAccessor.onException(e, Read); + throw e; + } + } + + private String executeGetDatasetVersionRenameStatement(PreparedStatement statement, int accountId, int containerId, + String datasetName, long sourceVersionValue) throws SQLException, AccountServiceException { + ResultSet resultSet = null; + try { + statement.setInt(1, accountId); + statement.setInt(2, containerId); + statement.setString(3, datasetName); + statement.setLong(4, sourceVersionValue); + resultSet = statement.executeQuery(); + if (!resultSet.next()) { + throw new AccountServiceException( + "Version not found for account: " + accountId + " container: " + containerId + " dataset: " + datasetName + + " version: " + sourceVersionValue, AccountServiceErrorCode.NotFound); + } + return resultSet.getString(RENAME_FROM); + } finally { + closeQuietly(resultSet); + } } @@ -480,6 +622,34 @@ public void deleteDatasetVersion(int accountId, int containerId, String datasetN } } + /** + * Delete a dataset version for dataset deletion. + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param version the version of the dataset. + * @throws SQLException + * @throws AccountServiceException + */ + public void deleteDatasetVersionForDatasetDelete(int accountId, int containerId, String datasetName, String version) + throws SQLException, AccountServiceException { + try { + long startTimeMs = System.currentTimeMillis(); + PreparedStatement getVersionSchemaStatement = dataAccessor.getPreparedStatement(getVersionSchemaSql, true); + Dataset.VersionSchema versionSchema = + executeGetVersionSchema(getVersionSchemaStatement, accountId, containerId, datasetName); + PreparedStatement deleteDatasetVersionStatement = + dataAccessor.getPreparedStatement(deleteDatasetVersionForDatasetDeleteByIdSql, true); + long versionValue = getVersionBasedOnSchema(version, versionSchema); + executeDeleteDatasetVersionForDatasetDeleteStatement(deleteDatasetVersionStatement, accountId, containerId, + datasetName, versionValue); + dataAccessor.onSuccess(Delete, System.currentTimeMillis() - startTimeMs); + } catch (SQLException | AccountServiceException e) { + dataAccessor.onException(e, Delete); + throw e; + } + } + /** * Update ttl for dataset version. * @param accountId the id for the parent account. @@ -816,10 +986,12 @@ private List executeListAllValidVersionsOrderedByLastModif resultSet = statement.executeQuery(); while (resultSet.next()) { long versionValue = resultSet.getLong(VERSION); + String renameFrom = resultSet.getString(RENAME_FROM); Timestamp deletionTime = resultSet.getTimestamp(DELETED_TS); String version = convertVersionValueToVersion(versionValue, versionSchema); datasetVersionRecordList.add( - new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime))); + new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime), + renameFrom)); } return datasetVersionRecordList; } finally { @@ -932,6 +1104,7 @@ private DatasetVersionRecord executeGetLatestVersionStatementForDownload(Prepare ResultSet resultSet = null; String version; Timestamp deletionTime; + String renameFrom; try { statement.setInt(1, accountId); statement.setInt(2, containerId); @@ -945,11 +1118,13 @@ private DatasetVersionRecord executeGetLatestVersionStatementForDownload(Prepare } long versionValue = resultSet.getLong(VERSION); version = convertVersionValueToVersion(versionValue, versionSchema); + renameFrom = resultSet.getString(RENAME_FROM); deletionTime = resultSet.getTimestamp(DELETED_TS); } finally { closeQuietly(resultSet); } - return new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime)); + return new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime), + renameFrom); } /** @@ -1107,9 +1282,11 @@ private List executeListAllValidVersionForDatasetDeletionS while (resultSet.next()) { long versionValue = resultSet.getLong(VERSION); String version = convertVersionValueToVersion(versionValue, versionSchema); + String renameFrom = resultSet.getString(RENAME_FROM); Timestamp deletionTime = resultSet.getTimestamp(DELETED_TS); DatasetVersionRecord datasetVersionRecord = - new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime)); + new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime), + renameFrom); datasetVersionRecords.add(datasetVersionRecord); } return datasetVersionRecords; @@ -1220,9 +1397,11 @@ private List executeListValidDatasetVersionsByListStatemen long versionValue = resultSet.getLong(VERSION); long expirationTimeMs = timestampToMs(resultSet.getTimestamp(DELETED_TS)); long creationTimeMs = timestampToMs(resultSet.getTimestamp(CREATION_TIME)); + String rename_from = resultSet.getString(RENAME_FROM); String version = convertVersionValueToVersion(versionValue, versionSchema); entries.add( - new DatasetVersionRecord(accountId, containerId, datasetName, version, expirationTimeMs, creationTimeMs)); + new DatasetVersionRecord(accountId, containerId, datasetName, version, expirationTimeMs, creationTimeMs, + rename_from)); } return entries; } finally { @@ -1369,6 +1548,28 @@ private void executeUpdateDatasetVersionStateStatement(PreparedStatement stateme statement.executeUpdate(); } + /** + * Execute the updateDatasetVersionStateAndDeletedTsSql statement. + * @param statement the updateDatasetVersionStateSql statement. + * @param statement the mysql statement to delete dataset version. + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param versionValue the version value of the dataset. + * @param datasetVersionState the {@link DatasetVersionState} + * @throws SQLException + */ + private void executeUpdateDatasetVersionStateAndDeletedStatement(PreparedStatement statement, int accountId, + int containerId, String datasetName, long versionValue, DatasetVersionState datasetVersionState) + throws SQLException { + statement.setInt(1, datasetVersionState.ordinal()); + statement.setInt(2, accountId); + statement.setInt(3, containerId); + statement.setString(4, datasetName); + statement.setLong(5, versionValue); + statement.executeUpdate(); + } + /** * Execute deleteDatasetVersionStatement to delete Dataset version. * @param statement the mysql statement to delete dataset version. @@ -1379,7 +1580,32 @@ private void executeUpdateDatasetVersionStateStatement(PreparedStatement stateme * @throws SQLException the {@link Dataset} including the metadata. * @throws AccountServiceException */ - private void executeDeleteDatasetVersionStatement(PreparedStatement statement, int accountId, + private void executeDeleteDatasetVersionStatement(PreparedStatement statement, int accountId, int containerId, + String datasetName, long versionValue) throws SQLException, AccountServiceException { + statement.setInt(1, accountId); + statement.setInt(2, containerId); + statement.setString(3, datasetName); + statement.setLong(4, versionValue); + statement.setInt(5, DatasetVersionState.RENAMED.ordinal()); + int count = statement.executeUpdate(); + if (count <= 0) { + throw new AccountServiceException( + "Dataset not found qualified record to delete for account: " + accountId + " container: " + containerId + + " dataset: " + datasetName + " version: " + versionValue, AccountServiceErrorCode.NotFound); + } + } + + /** + * Execute deleteDatasetVersionForDatasetDeleteByIdSql to delete Dataset version when delete a dataset. + * @param statement the mysql statement to delete dataset version. + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param versionValue the version value of the dataset. + * @throws SQLException + * @throws AccountServiceException + */ + private void executeDeleteDatasetVersionForDatasetDeleteStatement(PreparedStatement statement, int accountId, int containerId, String datasetName, long versionValue) throws SQLException, AccountServiceException { statement.setInt(1, accountId); statement.setInt(2, containerId); @@ -1393,6 +1619,35 @@ private void executeDeleteDatasetVersionStatement(PreparedStatement statement, i } } + /** + * Execute CopyToNewDatasetVersionStatement to copy the source version to a target version. + * @param statement the rename dataset statement. + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param sourceVersionValue the long value of source version. + * @param targetVersionValue the long value of target version. + * @throws SQLException + * @throws AccountServiceException + */ + private void executeCopyToNewDatasetVersionStatement(PreparedStatement statement, int accountId, int containerId, + String datasetName, long sourceVersionValue, long targetVersionValue, String renameFrom) + throws SQLException, AccountServiceException { + statement.setLong(1, targetVersionValue); + statement.setString(2, renameFrom); + statement.setInt(3, accountId); + statement.setInt(4, containerId); + statement.setString(5, datasetName); + statement.setLong(6, sourceVersionValue); + statement.setInt(7, DatasetVersionState.READY.ordinal()); + int count = statement.executeUpdate(); + if (count <= 0) { + throw new AccountServiceException( + "Can't find source version to rename for account: " + accountId + " container: " + containerId + " dataset: " + + datasetName + " version: " + sourceVersionValue, AccountServiceErrorCode.NotFound); + } + } + /** * Execute version ttl update statement. * @param statement the updateDatasetVersionTtlSql statement. @@ -1443,7 +1698,8 @@ private DatasetVersionRecord updateDatasetVersionHelper(int accountId, int conta executeUpdateDatasetVersionIfExpiredSqlStatement(updateDatasetVersionSqlIfExpiredStatement, accountId, containerId, dataset.getDatasetName(), versionNumber, timeToLiveInSeconds, creationTimeInMs, dataset, datasetVersionTtlEnabled, datasetVersionState); - return new DatasetVersionRecord(accountId, containerId, dataset.getDatasetName(), version, newExpirationTimeMs); + return new DatasetVersionRecord(accountId, containerId, dataset.getDatasetName(), version, newExpirationTimeMs, + null); } /** @@ -1784,6 +2040,7 @@ private DatasetVersionRecord executeGetDatasetVersionStatement(PreparedStatement throws SQLException, AccountServiceException { ResultSet resultSet = null; Timestamp deletionTime; + String renameFrom; try { statement.setInt(1, accountId); statement.setInt(2, containerId); @@ -1796,6 +2053,7 @@ private DatasetVersionRecord executeGetDatasetVersionStatement(PreparedStatement "Dataset version not found for account: " + accountId + " container: " + containerId + " dataset: " + datasetName + " version: " + version, AccountServiceErrorCode.NotFound); } + renameFrom = resultSet.getString(RENAME_FROM); deletionTime = resultSet.getTimestamp(DELETED_TS); long currentTime = System.currentTimeMillis(); if (compareTimestamp(deletionTime, currentTime) <= 0) { @@ -1806,7 +2064,8 @@ private DatasetVersionRecord executeGetDatasetVersionStatement(PreparedStatement } finally { closeQuietly(resultSet); } - return new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime)); + return new DatasetVersionRecord(accountId, containerId, datasetName, version, timestampToMs(deletionTime), + renameFrom); } @@ -1828,4 +2087,9 @@ private Dataset getDatasetHelper(int accountId, int containerId, String accountN return executeGetDatasetStatement(getDatasetStatement, accountId, containerId, accountName, containerName, datasetName); } + + private boolean isAutoIncrVersion(String version) { + final Set ALL_SEMANTIC_COMPONENTS = new HashSet<>(Arrays.asList(LATEST, MAJOR, MINOR, PATCH, REVISION)); + return ALL_SEMANTIC_COMPONENTS.contains(version); + } } diff --git a/ambry-account/src/main/java/com/github/ambry/account/mysql/MySqlAccountStore.java b/ambry-account/src/main/java/com/github/ambry/account/mysql/MySqlAccountStore.java index 84ac55402e..c62fdb12c6 100644 --- a/ambry-account/src/main/java/com/github/ambry/account/mysql/MySqlAccountStore.java +++ b/ambry-account/src/main/java/com/github/ambry/account/mysql/MySqlAccountStore.java @@ -199,6 +199,25 @@ public synchronized DatasetVersionRecord addDatasetVersion(int accountId, int co timeToLiveInSeconds, creationTimeInMs, datasetVersionTtlEnabled, datasetVersionState); } + /** + * Rename a dataset version of {@link Dataset} + * @param accountId the id for the parent account. + * @param containerId the id of the container. + * @param accountName the name for the parent account. + * @param containerName the name for the container. + * @param datasetName the name of the dataset. + * @param sourceVersion the source version to rename. + * @param targetVersion the target version to rename. + * @throws SQLException + * @throws AccountServiceException + */ + public synchronized void renameDatasetVersion(int accountId, int containerId, String accountName, + String containerName, String datasetName, String sourceVersion, String targetVersion) + throws SQLException, AccountServiceException { + datasetDao.renameDatasetVersion(accountId, containerId, accountName, containerName, datasetName, sourceVersion, + targetVersion); + } + public synchronized void updateDatasetVersionState(int accountId, int containerId, String accountName, String containerName, String datasetName, String version, DatasetVersionState datasetVersionState) throws SQLException, AccountServiceException { @@ -236,6 +255,11 @@ public synchronized void deleteDatasetVersion(short accountId, short containerId datasetDao.deleteDatasetVersion(accountId, containerId, datasetName, version); } + public synchronized void deleteDatasetVersionForDatasetDelete(short accountId, short containerId, String datasetName, + String version) throws SQLException, AccountServiceException { + datasetDao.deleteDatasetVersionForDatasetDelete(accountId, containerId, datasetName, version); + } + /** * Update ttl for a version of {@link Dataset} * @param accountId the id for the parent account. diff --git a/ambry-account/src/main/resources/AccountSchema.ddl b/ambry-account/src/main/resources/AccountSchema.ddl index a426a691d6..bd5e208784 100644 --- a/ambry-account/src/main/resources/AccountSchema.ddl +++ b/ambry-account/src/main/resources/AccountSchema.ddl @@ -73,6 +73,7 @@ CREATE TABLE IF NOT EXISTS DatasetVersions ( lastModifiedTime DATETIME(3) NOT NULL, delete_ts DATETIME(6) DEFAULT NULL, deleted_ts DATETIME(6) DEFAULT NULL, + rename_from VARCHAR(25) DEFAULT NULL, PRIMARY KEY (accountId, containerId, datasetName, version) ) CHARACTER SET utf8 COLLATE utf8_bin; diff --git a/ambry-api/src/main/java/com/github/ambry/account/AccountService.java b/ambry-api/src/main/java/com/github/ambry/account/AccountService.java index 2662266283..d26f18447d 100644 --- a/ambry-api/src/main/java/com/github/ambry/account/AccountService.java +++ b/ambry-api/src/main/java/com/github/ambry/account/AccountService.java @@ -304,6 +304,25 @@ default void deleteDatasetVersion(String accountName, String containerName, Stri throw new UnsupportedOperationException("This method is not supported"); } + default void deleteDatasetVersionForDatasetDelete(String accountName, String containerName, String datasetName, + String version) throws AccountServiceException { + throw new UnsupportedOperationException("This method is not supported"); + } + + /** + * Rename a dataset version. + * @param accountName The name for the parent account. + * @param containerName The name for the container. + * @param datasetName The name of the dataset. + * @param sourceVersion the source version which need to be renamed. + * @param targetVersion the target version to rename to. + * @throws AccountServiceException + */ + default void renameDatasetVersion(String accountName, String containerName, String datasetName, String sourceVersion, + String targetVersion) throws AccountServiceException { + throw new UnsupportedOperationException("This method is not supported"); + } + /** * Get all valid dataset versions for dataset deletion. * @param accountName The name for the parent account. diff --git a/ambry-api/src/main/java/com/github/ambry/account/DatasetVersionRecord.java b/ambry-api/src/main/java/com/github/ambry/account/DatasetVersionRecord.java index 6cb93bc2bc..858b851343 100644 --- a/ambry-api/src/main/java/com/github/ambry/account/DatasetVersionRecord.java +++ b/ambry-api/src/main/java/com/github/ambry/account/DatasetVersionRecord.java @@ -27,37 +27,46 @@ public class DatasetVersionRecord { private final String version; private final long expirationTimeMs; private final Long creationTimeMs; + private String renameFrom; + private static final String NAMED_BLOB_PREFIX = "/named"; + private static final String SLASH = "/"; + /** * Constructor that takes individual arguments. - * @param accountId the id of the parent account. - * @param containerId the id of the container. - * @param datasetName the name of the dataset. - * @param version the version of the dataset. + * + * @param accountId the id of the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param version the version of the dataset. * @param expirationTimeMs the expiration time in milliseconds since epoch, or -1 if the blob should be permanent. + * @param renameFrom the original version which renamed from */ public DatasetVersionRecord(int accountId, int containerId, String datasetName, String version, - long expirationTimeMs) { - this(accountId, containerId, datasetName, version, expirationTimeMs, null); + long expirationTimeMs, String renameFrom) { + this(accountId, containerId, datasetName, version, expirationTimeMs, null, renameFrom); } /** * Constructor for retention policy support. - * @param accountId the id of the parent account. - * @param containerId the id of the container. - * @param datasetName the name of the dataset. - * @param version the version of the dataset. + * + * @param accountId the id of the parent account. + * @param containerId the id of the container. + * @param datasetName the name of the dataset. + * @param version the version of the dataset. * @param expirationTimeMs the expiration time in milliseconds since epoch, or -1 if the blob should be permanent. - * @param creationTimeMs the creation time in milliseconds since epoch for dataset version. + * @param creationTimeMs the creation time in milliseconds since epoch for dataset version. + * @param renameFrom the original version which renamed from */ public DatasetVersionRecord(int accountId, int containerId, String datasetName, String version, long expirationTimeMs, - Long creationTimeMs) { + Long creationTimeMs, String renameFrom) { this.accountId = accountId; this.containerId = containerId; this.datasetName = datasetName; this.version = version; this.expirationTimeMs = expirationTimeMs; this.creationTimeMs = creationTimeMs; + this.renameFrom = renameFrom; } /** @@ -102,6 +111,30 @@ public long getCreationTimeMs() { return creationTimeMs; } + public String getRenameFrom() { + return renameFrom; + } + + private String getOriginalPath(String accountName, String containerName) { + return NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH + version; + } + + public String getRenamedPath(String accountName, String containerName) { + return NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH + renameFrom; + } + + public String getNamedBlobNamePath(String accountName, String containerName) { + if (this.renameFrom != null) { + return getRenamedPath(accountName, containerName); + } else { + return getOriginalPath(accountName, containerName); + } + } + + public void setRenameFrom(String renameFrom) { + this.renameFrom = renameFrom; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -113,7 +146,7 @@ public boolean equals(Object o) { DatasetVersionRecord record = (DatasetVersionRecord) o; return accountId == record.accountId && containerId == record.containerId && Objects.equals(datasetName, record.datasetName) && Objects.equals(version, record.version) && expirationTimeMs == record.expirationTimeMs - && Objects.equals(creationTimeMs, record.creationTimeMs); + && Objects.equals(creationTimeMs, record.creationTimeMs) && Objects.equals(renameFrom, record.renameFrom); } @Override diff --git a/ambry-api/src/main/java/com/github/ambry/frontend/DatasetVersionPath.java b/ambry-api/src/main/java/com/github/ambry/frontend/DatasetVersionPath.java index 9ab5dca924..dec736cb95 100644 --- a/ambry-api/src/main/java/com/github/ambry/frontend/DatasetVersionPath.java +++ b/ambry-api/src/main/java/com/github/ambry/frontend/DatasetVersionPath.java @@ -32,6 +32,13 @@ public class DatasetVersionPath { private final String containerName; private final String datasetName; private final String version; + private final String targetVersion; + public static final String RENAME = "RENAME"; + + public static final String OP = "op"; + public static final String TARGET_VERSION = "targetVersion"; + + public static DatasetVersionPath parse(RequestPath requestPath, Map args) throws RestServiceException { @@ -50,6 +57,7 @@ public static DatasetVersionPath parse(String path, Map args) th Objects.requireNonNull(path, "path should not be null"); Objects.requireNonNull(args, "args should not be null"); boolean isListRequest = RestUtils.getBooleanHeader(args, ENABLE_DATASET_VERSION_LISTING, false); + boolean isRenameRequest = "RENAME".equals(RestUtils.getHeader(args, OP, false)); path = path.startsWith("/") ? path.substring(1) : path; String[] splitPath = path.split("/"); int expectedMinimumSegments = isListRequest ? 4 : 5; @@ -65,25 +73,33 @@ public static DatasetVersionPath parse(String path, Map args) th //if isListRequest == false, the path format should be /named//// if (isListRequest) { datasetName = String.join("/", Arrays.copyOfRange(splitPath, 3, splitPath.length)); - return new DatasetVersionPath(accountName, containerName, datasetName, null); + return new DatasetVersionPath(accountName, containerName, datasetName, null, null); + } + String targetVersion = null; + if (isRenameRequest) { + targetVersion = RestUtils.getHeader(args, TARGET_VERSION, true); } datasetName = String.join("/", Arrays.copyOfRange(splitPath, 3, splitPath.length - 1)); String version = splitPath[splitPath.length - 1]; - return new DatasetVersionPath(accountName, containerName, datasetName, version); + return new DatasetVersionPath(accountName, containerName, datasetName, version, targetVersion); } /** * Construct a {@link DatasetVersionPath} - * @param accountName name of the parent account. + * + * @param accountName name of the parent account. * @param containerName name of the container. - * @param datasetName name of the dataset. - * @param version the version of the dataset. + * @param datasetName name of the dataset. + * @param version the version of the dataset. + * @param targetVersion the target version for copy version API. */ - private DatasetVersionPath(String accountName, String containerName, String datasetName, String version) { + private DatasetVersionPath(String accountName, String containerName, String datasetName, String version, + String targetVersion) { this.accountName = accountName; this.containerName = containerName; this.datasetName = datasetName; this.version = version; + this.targetVersion = targetVersion; } /** @@ -113,4 +129,6 @@ public String getDatasetName() { public String getVersion() { return version; } + + public String getTargetVersion() { return targetVersion; } } diff --git a/ambry-api/src/main/java/com/github/ambry/protocol/DatasetVersionState.java b/ambry-api/src/main/java/com/github/ambry/protocol/DatasetVersionState.java index 3add5bb6fd..2f5e3fd2ad 100644 --- a/ambry-api/src/main/java/com/github/ambry/protocol/DatasetVersionState.java +++ b/ambry-api/src/main/java/com/github/ambry/protocol/DatasetVersionState.java @@ -16,5 +16,6 @@ public enum DatasetVersionState { IN_PROGRESS, - READY + READY, + RENAMED } diff --git a/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java b/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java index b0ef9770ab..87dc7d41dc 100644 --- a/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java +++ b/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java @@ -198,6 +198,10 @@ public static final class Headers { * for put or get dataset request; long; version of dataset. */ public final static String TARGET_DATASET_VERSION = "x-ambry-target-dataset-version"; + /** + * The original named blob name for a certain dataset version. + */ + public final static String TARGET_DATASET_ORIGINAL_VERSION = "x-ambry-dataset-original-version"; /** * The dataset version expiration time. */ @@ -478,6 +482,12 @@ public static final class InternalKeys { */ public static final String REQUEST_PATH = KEY_PREFIX + "request-path"; + /** + * The internal header to determine if the delete request is coming from a dataset deletion. + */ + public static final String DATASET_DELETE_ENABLED = KEY_PREFIX + "dataset-delete-enabled"; + + /** * To be set to {@code true} if failures reason should be attached to frontend responses. */ diff --git a/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTest.java b/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTest.java index cba5d4ae09..519587a9d9 100644 --- a/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTest.java +++ b/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTest.java @@ -297,6 +297,33 @@ public void datasetTest() throws Exception { doDeleteDatasetAndVerify(refAccount.getName(), namedBlobOptionalContainer.getName(), datasetList); //After delete, it should have an empty list. doListDatasetAndVerify(refAccount.getName(), namedBlobOptionalContainer.getName(), new ArrayList<>()); + + } + + @Ignore + @Test + public void datasetRenameTest() throws Exception { + Account refAccount = ACCOUNT_SERVICE.createAndAddRandomAccount(); + Container namedBlobOptionalContainer = + new ContainerBuilder((short) 11, "optional", Container.ContainerStatus.ACTIVE, "", + refAccount.getId()).setNamedBlobMode(Container.NamedBlobMode.OPTIONAL).build(); + ACCOUNT_SERVICE.updateContainers(refAccount.getName(), Arrays.asList(namedBlobOptionalContainer)); + String contentType = "application/octet-stream"; + String ownerId = "datasetTest"; + int contentSize = 100; + List datasetList = addSemanticLongDataset(refAccount, namedBlobOptionalContainer, null); + List> allDatasetVersions = new ArrayList<>(); + List> datasetVersionsFromCopy = doCopyDatasetTestAndVerify(datasetList, contentType, ownerId, + contentSize); + allDatasetVersions.addAll(datasetVersionsFromCopy); + //Test List dataset version + List> allDatasetVersionPairs = doListDatasetVersionAndVerify(datasetList, allDatasetVersions); + doDatasetUpdateTtlAndVerify(refAccount.getName(), namedBlobOptionalContainer.getName(), allDatasetVersionPairs, + contentSize, contentType, ownerId); + //Test delete + doDeleteDatasetVersionAndVerify(refAccount.getName(), namedBlobOptionalContainer.getName(), allDatasetVersionPairs); + //After delete, it should have an empty list. + doListDatasetVersionAndVerify(datasetList, new ArrayList<>()); } @Ignore diff --git a/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTestBase.java b/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTestBase.java index 529b7e2b55..87d8631e0e 100644 --- a/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTestBase.java +++ b/ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTestBase.java @@ -85,8 +85,6 @@ import static com.github.ambry.frontend.Operations.*; import static com.github.ambry.rest.RestUtils.Headers.*; -import static com.github.ambry.frontend.Operations.*; -import static com.github.ambry.rest.RestUtils.Headers.*; import static com.github.ambry.utils.TestUtils.*; import static org.junit.Assert.*; @@ -473,6 +471,35 @@ List doDatasetPutUpdateGetTest(Account account, Container container, Lo return datasetList; } + List addSemanticLongDataset(Account account, Container container, Long ttl) throws Exception { + String accountName = account.getName(); + String containerName = container.getName(); + List versionSchemas = new ArrayList<>(); + List datasetList = new ArrayList<>(); + versionSchemas.add(SEMANTIC_LONG); + for (Dataset.VersionSchema versionSchema : versionSchemas) { + String datasetName = "zzzz" + TestUtils.getRandomString(10); + Dataset dataset; + if (ttl == null) { + dataset = new DatasetBuilder(accountName, containerName, datasetName).setVersionSchema(versionSchema).build(); + } else { + dataset = new DatasetBuilder(accountName, containerName, datasetName).setVersionSchema(versionSchema) + .setRetentionTimeInSeconds(ttl) + .build(); + } + HttpHeaders headers = new DefaultHttpHeaders(); + //Test put dataset + putDatasetAndVerify(dataset, headers, false); + HttpHeaders getHeaders = new DefaultHttpHeaders(); + getHeaders.add(TARGET_ACCOUNT_NAME, dataset.getAccountName()); + getHeaders.add(TARGET_CONTAINER_NAME, dataset.getContainerName()); + getHeaders.add(TARGET_DATASET_NAME, dataset.getDatasetName()); + getDatasetAndVerify(dataset, getHeaders); + datasetList.add(dataset); + } + return datasetList; + } + List> doDatasetVersionPutGetWithTtlTest(Account account, Container container, List datasets, String contentType, String ownerId, int contentSize) throws Exception { String accountName = account.getName(); @@ -539,6 +566,51 @@ void doDatasetUpdateTtlAndVerify(String accountName, String containerName, } } + List> doCopyDatasetTestAndVerify(List datasets, String contentType, String ownerId, + int contentSize) + throws Exception { + List> datasetVersions = new ArrayList<>(); + for (Dataset dataset : datasets) { + Dataset.VersionSchema versionSchema = dataset.getVersionSchema(); + if (SEMANTIC_LONG.equals(versionSchema)) { + long ttl = TTL_SECS; + String sourceVersion = generateDatasetVersion(dataset); + String targetVersion = generateDatasetVersion(dataset); + HttpHeaders headers = new DefaultHttpHeaders(); + headers.add(RestUtils.Headers.DATASET_VERSION_QUERY_ENABLED, true); + String accountName = dataset.getAccountName(); + String containerName = dataset.getContainerName(); + String datasetName = dataset.getDatasetName(); + setAmbryHeadersForPut(headers, ttl, false, accountName, contentType, ownerId, null, null); + ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(contentSize)); + putDatasetVersionAndVerify(dataset, sourceVersion, headers, content, contentSize, ttl); + + HttpHeaders expectedGetHeaders = new DefaultHttpHeaders().add(headers); + expectedGetHeaders.add(RestUtils.Headers.BLOB_SIZE, content.capacity()); + expectedGetHeaders.add(RestUtils.Headers.LIFE_VERSION, "0"); + expectedGetHeaders.add(TARGET_ACCOUNT_NAME, accountName); + expectedGetHeaders.add(RestUtils.Headers.TARGET_CONTAINER_NAME, containerName); + + headers = new DefaultHttpHeaders(); + headers.add(RestUtils.Headers.DATASET_VERSION_QUERY_ENABLED, true); + FullHttpRequest httpRequest = buildRequest(HttpMethod.PUT, + buildUriForDatasetVersionForCopy(accountName, containerName, datasetName, sourceVersion, targetVersion), + headers, null); + NettyClient.ResponseParts responseParts = nettyClient.sendRequest(httpRequest, null, null).get(); + HttpResponse response = getHttpResponse(responseParts); + assertEquals("Unexpected response status", HttpResponseStatus.OK, response.status()); + + HttpHeaders getHeaders = new DefaultHttpHeaders(); + getHeaders.add(RestUtils.Headers.DATASET_VERSION_QUERY_ENABLED, true); + + getDatasetVersionInfoAndVerify(accountName, containerName, dataset.getDatasetName(), targetVersion, getHeaders, ttl, + expectedGetHeaders); + datasetVersions.add(new Pair<>(dataset.getDatasetName(), targetVersion)); + } + } + return datasetVersions; + } + List> doDatasetVersionPutGetTest(Account account, Container container, List datasets, String contentType, String ownerId) throws Exception { String accountName = account.getName(); @@ -824,7 +896,7 @@ void getDatasetVersionInfoAndVerify(String accountName, String containerName, St if (expectTtl != -1) { assertEquals("Unexpected ttl value", expectTtl, (gmtToEpoch(response.headers().get(RestUtils.Headers.DATASET_EXPIRATION_TIME)) - gmtToEpoch( - response.headers().get(RestUtils.Headers.CREATION_TIME))) / 1000); + response.headers().get(RestUtils.Headers.CREATION_TIME))) / 1000); } assertTrue("Channel should be active", HttpUtil.isKeepAlive(response)); assertEquals(RestUtils.Headers.LIFE_VERSION + " does not match", @@ -926,6 +998,12 @@ String buildUriForDatasetVersion(String accountName, String containerName, Strin return String.format("/named/%s/%s/%s/%s", accountName, containerName, blobName, version); } + String buildUriForDatasetVersionForCopy(String accountName, String containerName, String blobName, + String sourceVersion, String targetVersion) { + return String.format("/named/%s/%s/%s/%s?op=RENAME&targetVersion=%s", accountName, containerName, blobName, + sourceVersion, targetVersion); + } + /** * The http request uri for named blob. * @param accountName The account name. diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/AccountAndContainerInjector.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/AccountAndContainerInjector.java index 608f13713b..442acbec5e 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/AccountAndContainerInjector.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/AccountAndContainerInjector.java @@ -441,6 +441,9 @@ private void setTargetDatasetAndVersionInRestRequestIfNeeded(RestRequest restReq } try { Dataset dataset = accountService.getDataset(accountName, containerName, datasetName); + if (restRequest.getArgs().get(InternalKeys.TARGET_DATASET) != null) { + return; + } restRequest.setArg(InternalKeys.TARGET_DATASET, dataset); if (version != null) { restRequest.setArg(InternalKeys.TARGET_DATASET_VERSION, version); diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/CopyDatasetVersionHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/CopyDatasetVersionHandler.java new file mode 100644 index 0000000000..32728ad402 --- /dev/null +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/CopyDatasetVersionHandler.java @@ -0,0 +1,164 @@ +/** + * Copyright 2024 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package com.github.ambry.frontend; + +import com.github.ambry.account.AccountService; +import com.github.ambry.account.AccountServiceException; +import com.github.ambry.commons.Callback; +import com.github.ambry.rest.RestRequest; +import com.github.ambry.rest.RestRequestMetrics; +import com.github.ambry.rest.RestResponseChannel; +import com.github.ambry.rest.RestServiceErrorCode; +import com.github.ambry.rest.RestServiceException; +import com.github.ambry.rest.RestUtils; +import com.github.ambry.router.ReadableStreamChannel; +import java.util.GregorianCalendar; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.github.ambry.frontend.FrontendUtils.*; + + +public class CopyDatasetVersionHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(CopyDatasetVersionHandler.class); + private final SecurityService securityService; + private final AccountAndContainerInjector accountAndContainerInjector; + private final FrontendMetrics frontendMetrics; + private final AccountService accountService; + + /** + * Constructs a handler for handling requests for copy dataset versions under a dataset. + * @param securityService the {@link SecurityService} to use. + * @param accountService the {@link AccountService} to use. + * @param frontendMetrics {@link FrontendMetrics} instance where metrics should be recorded. + * @param accountAndContainerInjector helper to resolve account and container for a given request. + */ + + CopyDatasetVersionHandler(SecurityService securityService, AccountService accountService, + FrontendMetrics frontendMetrics, AccountAndContainerInjector accountAndContainerInjector) { + this.securityService = securityService; + this.accountService = accountService; + this.frontendMetrics = frontendMetrics; + this.accountAndContainerInjector = accountAndContainerInjector; + } + + void handle(RestRequest restRequest, RestResponseChannel restResponseChannel, + Callback callback) throws RestServiceException { + RestRequestMetrics requestMetrics = + frontendMetrics.copyBlobMetricsGroup.getRestRequestMetrics(restRequest.isSslUsed(), false); + restRequest.getMetricsTracker().injectMetrics(requestMetrics); + // copy dataset request have their account/container name in request header, so checks can be done at early stage. + accountAndContainerInjector.injectAccountContainerForNamedBlob(restRequest, frontendMetrics.copyBlobMetricsGroup); + new CopyDatasetVersionHandler.CallbackChain(restRequest, restResponseChannel, callback).start(); + } + + /** + * Represents the chain of actions to take. Keeps request context that is relevant to all callback stages. + */ + + private class CallbackChain { + private final RestRequest restRequest; + private final String uri; + private final RestResponseChannel restResponseChannel; + private final Callback finalCallback; + + /** + * @param restRequest the {@link RestRequest}. + * @param restResponseChannel the {@link RestResponseChannel}. + * @param finalCallback the {@link Callback} to call on completion. + */ + + private CallbackChain(RestRequest restRequest, RestResponseChannel restResponseChannel, + Callback finalCallback) { + this.restRequest = restRequest; + this.restResponseChannel = restResponseChannel; + this.finalCallback = finalCallback; + this.uri = restRequest.getUri(); + } + + /** + * Start the chain by calling {@link SecurityService#preProcessRequest}. + */ + + private void start() { + // Start the callback chain by performing request security processing. + securityService.processRequest(restRequest, securityProcessRequestCallback()); + } + + /** + * After {@link SecurityService#processRequest} finishes, call {@link SecurityService#postProcessRequest} to perform + * request time security checks that rely on the request being fully parsed and any additional arguments set. + * @return a {@link Callback} to be used with {@link SecurityService#processRequest}. + */ + + private Callback securityProcessRequestCallback() { + return buildCallback(frontendMetrics.getDatasetsSecurityProcessRequestMetrics, + securityCheckResult -> securityService.postProcessRequest(restRequest, securityPostProcessRequestCallback()), + uri, LOGGER, finalCallback); + } + + /** + * After {@link SecurityService#processRequest} finishes, call {@link SecurityService#postProcessRequest} to perform + * request time security checks that rely on the request being fully parsed and any additional arguments set. + * @return a {@link Callback} to be used with {@link SecurityService#processRequest}. + */ + + private Callback securityPostProcessRequestCallback() { + return buildCallback(frontendMetrics.copyDatasetsSecurityPostProcessRequestMetrics, securityCheckResult -> { + LOGGER.debug("Received request for copy a dataset version with arguments: {}", restRequest.getArgs()); + frontendMetrics.copyDatasetVersionRate.mark(); + copyDatasetVersions(); + restResponseChannel.setHeader(RestUtils.Headers.DATE, new GregorianCalendar().getTime()); + restResponseChannel.setHeader(RestUtils.Headers.CONTENT_LENGTH, 0); + finalCallback.onCompletion(null, null); + }, uri, LOGGER, finalCallback); + } + + /** + * List all valid dataset versions under the specific dataset. + * @return the page of the dataset version. + * @throws RestServiceException + */ + + private void copyDatasetVersions() throws RestServiceException { + long startListDatasetVersionTime = System.currentTimeMillis(); + String accountName = null; + String containerName = null; + String sourceDatasetVersion = null; + String targetDatasetVersion = null; + String datasetName = null; + try { + DatasetVersionPath datasetVersionPath = + DatasetVersionPath.parse(RestUtils.getRequestPath(restRequest), restRequest.getArgs()); + accountName = datasetVersionPath.getAccountName(); + containerName = datasetVersionPath.getContainerName(); + datasetName = datasetVersionPath.getDatasetName(); + sourceDatasetVersion = datasetVersionPath.getVersion(); + targetDatasetVersion = datasetVersionPath.getTargetVersion(); + accountService.renameDatasetVersion(accountName, containerName, datasetName, sourceDatasetVersion, + targetDatasetVersion); + frontendMetrics.copyDatasetVersionProcessingTimeInMs.update( + System.currentTimeMillis() - startListDatasetVersionTime); + } catch (AccountServiceException ex) { + LOGGER.error("Dataset version rename failed for accountName " + accountName + " containerName " + containerName + + " datasetName " + datasetName + " sourceDatasetVersion " + sourceDatasetVersion + " targetDatasetVersion " + + targetDatasetVersion); + frontendMetrics.copyDatasetVersionError.inc(); + throw new RestServiceException(ex.getMessage(), + RestServiceErrorCode.getRestServiceErrorCode(ex.getErrorCode())); + } + } + } +} diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteBlobHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteBlobHandler.java index 2d02bbb0b7..9b283d2cd6 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteBlobHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteBlobHandler.java @@ -13,12 +13,16 @@ */ package com.github.ambry.frontend; +import com.github.ambry.account.Account; import com.github.ambry.account.AccountService; import com.github.ambry.account.AccountServiceException; +import com.github.ambry.account.Container; import com.github.ambry.account.Dataset; +import com.github.ambry.account.DatasetVersionRecord; import com.github.ambry.clustermap.ClusterMap; import com.github.ambry.commons.BlobId; import com.github.ambry.commons.Callback; +import com.github.ambry.named.NamedBlobRecord; import com.github.ambry.quota.QuotaManager; import com.github.ambry.quota.QuotaUtils; import com.github.ambry.rest.RequestPath; @@ -135,6 +139,19 @@ private Callback securityProcessRequestCallback() { private Callback securityPostProcessRequestCallback() { return buildCallback(metrics.deleteBlobSecurityPostProcessRequestMetrics, result -> { String serviceId = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.SERVICE_ID, false); + if (RestUtils.isDatasetVersionQueryEnabled(restRequest.getArgs())) { + accountAndContainerInjector.injectDatasetForNamedBlob(restRequest); + String datasetVersionPathString = getRequestPath(restRequest).getOperationOrBlobId(false); + DatasetVersionRecord datasetVersionRecord = + getDatasetVersionHelper(restRequest, datasetVersionPathString, accountService, metrics); + if (datasetVersionRecord.getRenameFrom() != null) { + RequestPath requestPath = getRequestPath(restRequest); + RequestPath newRequestPath = reconstructRequestPath(datasetVersionRecord, requestPath, + ((Account) restRequest.getArgs().get(InternalKeys.TARGET_ACCOUNT_KEY)).getName(), + ((Container) restRequest.getArgs().get(InternalKeys.TARGET_CONTAINER_KEY)).getName()); + restRequest.setArg(InternalKeys.REQUEST_PATH, newRequestPath); + } + } router.deleteBlob(restRequest, null, serviceId, routerCallback(), QuotaUtils.buildQuotaChargeCallback(restRequest, quotaManager, false)); }, restRequest.getUri(), LOGGER, finalCallback); @@ -149,7 +166,6 @@ private Callback routerCallback() { if (RestUtils.isDatasetVersionQueryEnabled(restRequest.getArgs())) { try { metrics.deleteDatasetVersionRate.mark(); - accountAndContainerInjector.injectDatasetForNamedBlob(restRequest); deleteDatasetVersion(restRequest); } catch (RestServiceException e) { metrics.deleteDatasetVersionError.inc(); @@ -207,7 +223,11 @@ private void deleteDatasetVersion(RestRequest restRequest) throws RestServiceExc containerName = dataset.getContainerName(); datasetName = dataset.getDatasetName(); version = (String) restRequest.getArgs().get(TARGET_DATASET_VERSION); - accountService.deleteDatasetVersion(accountName, containerName, datasetName, version); + if (restRequest.getArgs().get(DATASET_DELETE_ENABLED) != null) { + accountService.deleteDatasetVersionForDatasetDelete(accountName, containerName, datasetName, version); + } else { + accountService.deleteDatasetVersion(accountName, containerName, datasetName, version); + } LOGGER.debug( "Successfully deleteDataset version for accountName: " + accountName + " containerName: " + containerName + " datasetName: " + datasetName + " version: " + version); diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteDatasetHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteDatasetHandler.java index 8cfaf67337..7a74888f1f 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteDatasetHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/DeleteDatasetHandler.java @@ -145,16 +145,8 @@ private void deleteDataset() throws RestServiceException { List datasetVersionRecordList = accountService.getAllValidVersionForDatasetDeletion(accountName, containerName, datasetName); if (datasetVersionRecordList.size() > 0) { - DatasetVersionRecord record = datasetVersionRecordList.get(0); - String version = record.getVersion(); - RequestPath requestPath = getRequestPath(restRequest); - RequestPath newRequestPath = - new RequestPath(requestPath.getPrefix(), requestPath.getClusterName(), requestPath.getPathAfterPrefixes(), - NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH - + version, requestPath.getSubResource(), requestPath.getBlobSegmentIdx()); - // Replace RequestPath in the RestRequest and call DeleteBlobHandler.handle. - restRequest.setArg(InternalKeys.REQUEST_PATH, newRequestPath); - restRequest.setArg(Headers.DATASET_VERSION_QUERY_ENABLED, "true"); + reconstructRestRequest(restRequest, datasetVersionRecordList.get(0), accountName, containerName); + restRequest.setArg(InternalKeys.DATASET_DELETE_ENABLED, "true"); deleteBlobHandler.handle(restRequest, restResponseChannel, recursiveCallback(datasetVersionRecordList, 1, accountName, containerName, datasetName)); } else { @@ -197,15 +189,7 @@ private Callback recursiveCallback(List datasetVersi recursiveCallback(datasetVersionRecordList, idx + 1, accountName, containerName, datasetName); DatasetVersionRecord record = datasetVersionRecordList.get(idx); return buildCallback(frontendMetrics.deleteBlobSecurityProcessResponseMetrics, securityCheckResult -> { - String version = record.getVersion(); - RequestPath requestPath = getRequestPath(restRequest); - RequestPath newRequestPath = - new RequestPath(requestPath.getPrefix(), requestPath.getClusterName(), requestPath.getPathAfterPrefixes(), - NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH + version, - requestPath.getSubResource(), requestPath.getBlobSegmentIdx()); - // Replace RequestPath in the RestRequest and call DeleteBlobHandler.handle. - restRequest.setArg(InternalKeys.REQUEST_PATH, newRequestPath); - restRequest.setArg(Headers.DATASET_VERSION_QUERY_ENABLED, "true"); + reconstructRestRequest(restRequest, record, accountName, containerName); deleteBlobHandler.handle(restRequest, restResponseChannel, nextCallBack); }, uri, LOGGER, finalCallback); } diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendMetrics.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendMetrics.java index ce7da804a5..a71c9ffa40 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendMetrics.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendMetrics.java @@ -36,6 +36,8 @@ public class FrontendMetrics { // DELETE public final RestRequestMetricsGroup deleteBlobMetricsGroup; public final RestRequestMetricsGroup deleteDatasetsMetricsGroup; + //COPY + public final RestRequestMetricsGroup copyBlobMetricsGroup; // GET public final RestRequestMetricsGroup getBlobMetricsGroup; @@ -130,6 +132,8 @@ public class FrontendMetrics { public final AsyncOperationTracker.Metrics getDatasetsSecurityProcessRequestMetrics; public final AsyncOperationTracker.Metrics getDatasetsSecurityPostProcessRequestMetrics; + public final AsyncOperationTracker.Metrics copyDatasetsSecurityPostProcessRequestMetrics; + public final AsyncOperationTracker.Metrics listDatasetsSecurityProcessRequestMetrics; public final AsyncOperationTracker.Metrics listDatasetsSecurityPostProcessRequestMetrics; @@ -281,6 +285,7 @@ public class FrontendMetrics { public final Counter deleteDatasetVersionError; public final Counter ttlUpdateDatasetVersionError; public final Counter listDatasetVersionError; + public final Counter copyDatasetVersionError; public final Counter deleteDatasetVersionOutOfRetentionError; public final Counter deleteDatasetVersionIfUploadFailCount; public final Meter addDatasetVersionRate; @@ -288,11 +293,15 @@ public class FrontendMetrics { public final Meter deleteDatasetVersionRate; public final Meter updateTtlDatasetVersionRate; public final Meter listDatasetVersionRate; + public final Meter copyDatasetVersionRate; + public final Histogram addDatasetVersionProcessingTimeInMs; public final Histogram getDatasetVersionProcessingTimeInMs; public final Histogram deleteDatasetVersionProcessingTimeInMs; public final Histogram updateTtlDatasetVersionProcessingTimeInMs; public final Histogram listDatasetVersionProcessingTimeInMs; + public final Histogram copyDatasetVersionProcessingTimeInMs; + private final MetricRegistry metricRegistry; /** @@ -309,6 +318,9 @@ public FrontendMetrics(MetricRegistry metricRegistry, FrontendConfig frontendCon deleteDatasetsMetricsGroup = new RestRequestMetricsGroup(FrontendRestRequestService.class, "DeleteDataset", false, metricRegistry, frontendConfig); + //COPY + copyBlobMetricsGroup = + new RestRequestMetricsGroup(FrontendRestRequestService.class, "CopyBlob", true, metricRegistry, frontendConfig); // GET getBlobMetricsGroup = new RestRequestMetricsGroup(FrontendRestRequestService.class, "GetBlob", true, metricRegistry, frontendConfig); @@ -476,6 +488,8 @@ public FrontendMetrics(MetricRegistry metricRegistry, FrontendConfig frontendCon new AsyncOperationTracker.Metrics(GetDatasetsHandler.class, "SecurityProcessRequest", metricRegistry); getDatasetsSecurityPostProcessRequestMetrics = new AsyncOperationTracker.Metrics(GetDatasetsHandler.class, "SecurityPostProcessRequest", metricRegistry); + copyDatasetsSecurityPostProcessRequestMetrics = + new AsyncOperationTracker.Metrics(GetDatasetsHandler.class, "SecurityPostProcessRequest", metricRegistry); listDatasetsSecurityProcessRequestMetrics = new AsyncOperationTracker.Metrics(GetDatasetsHandler.class, "SecurityProcessRequest", metricRegistry); @@ -725,6 +739,8 @@ public FrontendMetrics(MetricRegistry metricRegistry, FrontendConfig frontendCon metricRegistry.counter(MetricRegistry.name(TtlUpdateHandler.class, "TtlUpdateDatasetVersionError")); listDatasetVersionError = metricRegistry.counter(MetricRegistry.name(ListDatasetVersionHandler.class, "ListDatasetVersionError")); + copyDatasetVersionError = + metricRegistry.counter(MetricRegistry.name(ListDatasetVersionHandler.class, "CopyDatasetVersionError")); deleteDatasetVersionOutOfRetentionError = metricRegistry.counter( MetricRegistry.name(NamedBlobPutHandler.class, "DeleteDatasetVersionOutOfRetentionError")); deleteDatasetVersionIfUploadFailCount = @@ -738,6 +754,8 @@ public FrontendMetrics(MetricRegistry metricRegistry, FrontendConfig frontendCon metricRegistry.meter(MetricRegistry.name(TtlUpdateHandler.class, "UpdateTtlDatasetVersionRate")); listDatasetVersionRate = metricRegistry.meter(MetricRegistry.name(ListDatasetVersionHandler.class, "ListDatasetVersionRate")); + copyDatasetVersionRate = + metricRegistry.meter(MetricRegistry.name(ListDatasetVersionHandler.class, "CopyDatasetVersionRate")); addDatasetVersionProcessingTimeInMs = metricRegistry.histogram(MetricRegistry.name(NamedBlobPutHandler.class, "AddDatasetVersionProcessingTimeInMs")); getDatasetVersionProcessingTimeInMs = @@ -748,6 +766,8 @@ public FrontendMetrics(MetricRegistry metricRegistry, FrontendConfig frontendCon MetricRegistry.name(TtlUpdateHandler.class, "updateTtlDatasetVersionProcessingTimeInMs")); listDatasetVersionProcessingTimeInMs = metricRegistry.histogram( MetricRegistry.name(ListDatasetVersionHandler.class, "ListDatasetVersionProcessingTimeInMs")); + copyDatasetVersionProcessingTimeInMs = metricRegistry.histogram( + MetricRegistry.name(ListDatasetVersionHandler.class, "CopyDatasetVersionProcessingTimeInMs")); this.metricRegistry = metricRegistry; } diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendRestRequestService.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendRestRequestService.java index 61f1185413..123e1ec373 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendRestRequestService.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendRestRequestService.java @@ -55,6 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.github.ambry.frontend.DatasetVersionPath.*; import static com.github.ambry.frontend.Operations.*; import static com.github.ambry.rest.RestUtils.*; import static com.github.ambry.rest.RestUtils.Headers.*; @@ -97,6 +98,7 @@ class FrontendRestRequestService implements RestRequestService { private GetBlobHandler getBlobHandler; private PostBlobHandler postBlobHandler; private TtlUpdateHandler ttlUpdateHandler; + private CopyDatasetVersionHandler copyDatasetVersionHandler; private DeleteBlobHandler deleteBlobHandler; private DeleteDatasetHandler deleteDatasetHandler; private HeadBlobHandler headBlobHandler; @@ -200,6 +202,8 @@ public void start() throws InstantiationException { ttlUpdateHandler = new TtlUpdateHandler(router, securityService, idConverter, accountAndContainerInjector, frontendMetrics, clusterMap, quotaManager, namedBlobDb, accountService); + copyDatasetVersionHandler = + new CopyDatasetVersionHandler(securityService, accountService, frontendMetrics, accountAndContainerInjector); deleteBlobHandler = new DeleteBlobHandler(router, securityService, idConverter, accountAndContainerInjector, frontendMetrics, clusterMap, quotaManager, accountService); @@ -378,6 +382,10 @@ public void handlePut(RestRequest restRequest, RestResponseChannel restResponseC if (isS3Request(restRequest)) { s3PutHandler.handle(restRequest, restResponseChannel, (r, e) -> submitResponse(restRequest, restResponseChannel, null, e)); + } else if (RestUtils.isDatasetVersionQueryEnabled(restRequest.getArgs()) + && DatasetVersionPath.parse(requestPath, restRequest.getArgs()).getTargetVersion() != null) { + copyDatasetVersionHandler.handle(restRequest, restResponseChannel, + (r, e) -> submitResponse(restRequest, restResponseChannel, null, e)); } else { namedBlobPutHandler.handle(restRequest, restResponseChannel, (r, e) -> submitResponse(restRequest, restResponseChannel, null, e)); diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendUtils.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendUtils.java index fe1654eec1..ece6031582 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendUtils.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/FrontendUtils.java @@ -13,6 +13,8 @@ */ package com.github.ambry.frontend; +import com.github.ambry.account.AccountService; +import com.github.ambry.account.AccountServiceException; import com.github.ambry.account.DatasetVersionRecord; import com.github.ambry.clustermap.ClusterMap; import com.github.ambry.commons.BlobId; @@ -39,6 +41,7 @@ import org.json.JSONObject; import org.json.JSONTokener; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.github.ambry.rest.RestUtils.*; import static com.github.ambry.rest.RestUtils.InternalKeys.*; @@ -50,6 +53,8 @@ public class FrontendUtils { static final String NAMED_BLOB_PREFIX = "/named"; static final String SLASH = "/"; + private static final Logger LOGGER = LoggerFactory.getLogger(FrontendUtils.class); + /** * Throws the specified {@link RestServiceException} if isEnabled is {@code true}. No-op otherwise. @@ -182,4 +187,47 @@ static void replaceRequestPathWithNewOperationOrBlobIdIfNeeded(RestRequest restR restRequest.setArg(RestUtils.InternalKeys.REQUEST_PATH, newRequestPath); } } + + public static DatasetVersionRecord getDatasetVersionHelper(RestRequest restRequest, String datasetVersionPathString, AccountService accountService, FrontendMetrics metrics) throws RestServiceException { + DatasetVersionPath datasetVersionPath = DatasetVersionPath.parse(datasetVersionPathString, restRequest.getArgs()); + String accountName = datasetVersionPath.getAccountName(); + String containerName = datasetVersionPath.getContainerName(); + String datasetName = datasetVersionPath.getDatasetName(); + String version = datasetVersionPath.getVersion(); + try { + return accountService.getDatasetVersion(accountName, containerName, datasetName, version); + } catch (AccountServiceException ex) { + LOGGER.error("Dataset version get failed for accountName: " + accountName + " containerName: " + containerName + + " datasetName: " + datasetName + " version: " + version, ex); + metrics.getDatasetVersionError.inc(); + throw new RestServiceException(ex.getMessage(), + RestServiceErrorCode.getRestServiceErrorCode(ex.getErrorCode())); + } + } + + public static RequestPath reconstructRequestPath(DatasetVersionRecord record, RequestPath requestPath, + String accountName, String containerName) { + RequestPath newRequestPath; + newRequestPath = + new RequestPath(requestPath.getPrefix(), requestPath.getClusterName(), requestPath.getPathAfterPrefixes(), + record.getNamedBlobNamePath(accountName, containerName), requestPath.getSubResource(), + requestPath.getBlobSegmentIdx()); + return newRequestPath; + } + + public static void reconstructRestRequest(RestRequest restRequest, DatasetVersionRecord record, String accountName, + String containerName) { + RequestPath requestPath = getRequestPath(restRequest); + RequestPath newRequestPath = reconstructRequestPath(record, requestPath, accountName, containerName); + // Replace RequestPath in the RestRequest and call DeleteBlobHandler.handle. + restRequest.setArg(InternalKeys.REQUEST_PATH, newRequestPath); + restRequest.setArg(Headers.DATASET_VERSION_QUERY_ENABLED, "true"); + if (restRequest.getArgs().get(InternalKeys.TARGET_ACCOUNT_KEY) != null) { + restRequest.setArg(InternalKeys.TARGET_ACCOUNT_KEY, null); + } + if (restRequest.getArgs().get(InternalKeys.TARGET_CONTAINER_KEY) != null) { + restRequest.setArg(InternalKeys.TARGET_CONTAINER_KEY, null); + } + restRequest.setArg(Headers.DATASET_VERSION_QUERY_ENABLED, "true"); + } } diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/GetBlobHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/GetBlobHandler.java index f92cf1f4ce..1ddd9c1bfc 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/GetBlobHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/GetBlobHandler.java @@ -342,14 +342,17 @@ private String getDatasetVersion(RestRequest restRequest) throws RestServiceExce restResponseChannel.setHeader(RestUtils.Headers.TARGET_CONTAINER_NAME, containerName); restResponseChannel.setHeader(RestUtils.Headers.TARGET_DATASET_NAME, datasetName); restResponseChannel.setHeader(RestUtils.Headers.TARGET_DATASET_VERSION, datasetVersionRecord.getVersion()); + if (datasetVersionRecord.getRenameFrom() != null) { + restResponseChannel.setHeader(RestUtils.Headers.TARGET_DATASET_ORIGINAL_VERSION, + datasetVersionRecord.getRenameFrom()); + } if (datasetVersionRecord.getExpirationTimeMs() != Utils.Infinite_Time) { restResponseChannel.setHeader(RestUtils.Headers.DATASET_EXPIRATION_TIME, new Date(datasetVersionRecord.getExpirationTimeMs())); } metrics.getDatasetVersionProcessingTimeInMs.update(System.currentTimeMillis() - startGetDatasetVersionTime); // If version is null, use the latest version + 1 from DatasetVersionRecord to construct named blob path. - return NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH - + datasetVersionRecord.getVersion(); + return datasetVersionRecord.getNamedBlobNamePath(accountName, containerName); } catch (AccountServiceException ex) { LOGGER.error( "Failed to get dataset version for accountName: " + accountName + " containerName: " + containerName diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobPutHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobPutHandler.java index f393f1ce1e..eb4d210591 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobPutHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobPutHandler.java @@ -742,21 +742,8 @@ private void recursiveDeleteDatasetVersions(RestRequest restRequest, return; } Dataset dataset = (Dataset) restRequest.getArgs().get(RestUtils.InternalKeys.TARGET_DATASET); - String accountName = dataset.getAccountName(); - String containerName = dataset.getContainerName(); - String datasetName = dataset.getDatasetName(); - DatasetVersionRecord record = datasetVersionRecordList.get(idx); - String version = record.getVersion(); - RequestPath requestPath = getRequestPath(restRequest); - RequestPath newRequestPath = - new RequestPath(requestPath.getPrefix(), requestPath.getClusterName(), requestPath.getPathAfterPrefixes(), - NAMED_BLOB_PREFIX + SLASH + accountName + SLASH + containerName + SLASH + datasetName + SLASH + version, - requestPath.getSubResource(), requestPath.getBlobSegmentIdx()); - LOGGER.debug("New request path : " + newRequestPath); - // Replace RequestPath in the WrappedRestRequest for delete and call DeleteBlobHandler.handle. - restRequest.setArg(InternalKeys.REQUEST_PATH, newRequestPath); - restRequest.setArg(InternalKeys.TARGET_ACCOUNT_KEY, null); - restRequest.setArg(InternalKeys.TARGET_CONTAINER_KEY, null); + reconstructRestRequest(restRequest, datasetVersionRecordList.get(idx), dataset.getAccountName(), + dataset.getContainerName()); //for delete out of retention request, we don't want to set anything to response channel. deleteBlobHandler.handle(restRequest, new NoOpResponseChannel(), (r, e) -> { if (e != null) { diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/TtlUpdateHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/TtlUpdateHandler.java index 7af1e99654..af83a0b901 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/TtlUpdateHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/TtlUpdateHandler.java @@ -13,8 +13,11 @@ */ package com.github.ambry.frontend; +import com.github.ambry.account.Account; import com.github.ambry.account.AccountService; import com.github.ambry.account.AccountServiceException; +import com.github.ambry.account.Container; +import com.github.ambry.account.DatasetVersionRecord; import com.github.ambry.clustermap.ClusterMap; import com.github.ambry.commons.BlobId; import com.github.ambry.commons.Callback; @@ -38,6 +41,7 @@ import org.slf4j.LoggerFactory; import static com.github.ambry.frontend.FrontendUtils.*; +import static com.github.ambry.rest.RestUtils.*; import static com.github.ambry.rest.RestUtils.InternalKeys.*; @@ -133,6 +137,16 @@ private void start() { private Callback securityProcessRequestCallback() { return buildCallback(metrics.updateBlobTtlSecurityProcessRequestMetrics, result -> { blobIdStr = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.BLOB_ID, true); + if (RestUtils.isDatasetVersionQueryEnabled(restRequest.getArgs())) { + String datasetVersionPathString = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.BLOB_ID, true); + DatasetVersionRecord datasetVersionRecord = + getDatasetVersionHelper(restRequest, datasetVersionPathString, accountService, metrics); + if (datasetVersionRecord.getRenameFrom() != null) { + DatasetVersionPath datasetVersionPath = DatasetVersionPath.parse(blobIdStr, restRequest.getArgs()); + blobIdStr = datasetVersionRecord.getRenamedPath(datasetVersionPath.getAccountName(), + datasetVersionPath.getContainerName()); + } + } idConverter.convert(restRequest, blobIdStr, idConverterCallback()); }, restRequest.getUri(), LOGGER, finalCallback); } diff --git a/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java b/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java index ecdad14705..02c10a869b 100644 --- a/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java +++ b/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java @@ -25,6 +25,7 @@ import com.github.ambry.account.ContainerBuilder; import com.github.ambry.account.Dataset; import com.github.ambry.account.DatasetBuilder; +import com.github.ambry.account.DatasetVersionRecord; import com.github.ambry.account.InMemAccountService; import com.github.ambry.account.InMemAccountServiceFactory; import com.github.ambry.accountstats.AccountStatsStore; @@ -725,6 +726,98 @@ public void testDatasetVersionTtlUpdate() throws Exception { assertNull("Mismatch on expiration time", restResponseChannel.getResponseHeaders().get(DATASET_EXPIRATION_TIME)); } + @Test + public void testRenameDatasetVersion() throws Exception { + //Add dataset + Account testAccount = new ArrayList<>(accountService.getAllAccounts()).get(1); + Container testContainer = new ArrayList<>(testAccount.getAllContainers()).get(1); + Dataset.VersionSchema versionSchema = Dataset.VersionSchema.SEMANTIC_LONG; + Dataset dataset = + new DatasetBuilder(testAccount.getName(), testContainer.getName(), DATASET_NAME).setVersionSchema(versionSchema) + .build(); + byte[] datasetsUpdateJson = AccountCollectionSerde.serializeDatasetsInJson(dataset); + List body = new LinkedList<>(); + body.add(ByteBuffer.wrap(datasetsUpdateJson)); + body.add(null); + JSONObject headers = new JSONObject().put(RestUtils.Headers.TARGET_ACCOUNT_NAME, testAccount.getName()) + .put(RestUtils.Headers.TARGET_CONTAINER_NAME, testContainer.getName()); + RestRequest restRequest = + createRestRequest(RestMethod.POST, Operations.ACCOUNTS_CONTAINERS_DATASETS, headers, body); + MockRestResponseChannel restResponseChannel = new MockRestResponseChannel(); + doOperation(restRequest, restResponseChannel); + + // add dataset version + String version = "1.1.1.1"; + String blobName = DATASET_NAME + SLASH + version; + String namedBlobPathUri = + NAMED_BLOB_PREFIX + SLASH + testAccount.getName() + SLASH + testContainer.getName() + SLASH + blobName; + ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(10)); + body = new LinkedList<>(); + body.add(content); + body.add(null); + headers = new JSONObject(); + setAmbryHeadersForPut(headers, -1, testContainer.isCacheable(), "test", "application/octet-stream", "owner", null, + null, null); + headers.put(RestUtils.Headers.DATASET_VERSION_QUERY_ENABLED, true); + restRequest = createRestRequest(RestMethod.PUT, namedBlobPathUri, headers, body); + restResponseChannel = new MockRestResponseChannel(); + + BlobProperties blobProperties = + new BlobProperties(0, testAccount.getName(), "owner", "image/gif", false, 7200, testAccount.getId(), + testContainer.getId(), false, null, null, null); + ReadableStreamChannel byteBufferContent = new ByteBufferReadableStreamChannel(ByteBuffer.allocate(10)); + String blobIdFromRouter = + router.putBlobWithIdVersion(blobProperties, new byte[0], byteBufferContent, BlobId.BLOB_ID_V6).get(); + + reset(namedBlobDb); + NamedBlobRecord namedBlobRecord = + new NamedBlobRecord(testAccount.getName(), testContainer.getName(), blobName, blobIdFromRouter, 3600); + when(namedBlobDb.put(any(), any(), any())).thenReturn( + CompletableFuture.completedFuture(new PutResult(namedBlobRecord))); + when(namedBlobDb.delete(namedBlobRecord.getAccountName(), namedBlobRecord.getContainerName(), blobName)).thenReturn( + CompletableFuture.completedFuture(new DeleteResult(blobIdFromRouter, false))); + when(namedBlobDb.get(namedBlobRecord.getAccountName(), namedBlobRecord.getContainerName(), blobName, + GetOption.None)).thenReturn(CompletableFuture.completedFuture(namedBlobRecord)); + when(namedBlobDb.updateBlobTtlAndStateToReady(any())).thenReturn( + CompletableFuture.completedFuture(new PutResult(namedBlobRecord))); + doOperation(restRequest, restResponseChannel); + + // rename dataset version + String newVersion = "10.10.10.10"; + String blobNewName = DATASET_NAME + SLASH + newVersion; + namedBlobPathUri = + NAMED_BLOB_PREFIX + SLASH + testAccount.getName() + SLASH + testContainer.getName() + SLASH + blobName + + "?op=RENAME&targetVersion=" + newVersion; + headers.put(RestUtils.Headers.DATASET_VERSION_QUERY_ENABLED, true); + restRequest = createRestRequest(RestMethod.PUT, namedBlobPathUri, headers, body); + restResponseChannel = new MockRestResponseChannel(); + + reset(namedBlobDb); + NamedBlobRecord newNamedBlobRecord = + new NamedBlobRecord(testAccount.getName(), testContainer.getName(), blobNewName, blobIdFromRouter, 3600); + when(namedBlobDb.get(namedBlobRecord.getAccountName(), namedBlobRecord.getContainerName(), blobNewName, + GetOption.None)).thenReturn(CompletableFuture.completedFuture(newNamedBlobRecord)); + when(namedBlobDb.get(namedBlobRecord.getAccountName(), namedBlobRecord.getContainerName(), blobName, + GetOption.None)).thenThrow(new RuntimeException()); + when(namedBlobDb.delete(namedBlobRecord.getAccountName(), namedBlobRecord.getContainerName(), blobName)).thenReturn( + CompletableFuture.completedFuture(new DeleteResult(blobIdFromRouter, false))); + when(namedBlobDb.updateBlobTtlAndStateToReady(any())).thenReturn( + CompletableFuture.completedFuture(new PutResult(namedBlobRecord))); + doOperation(restRequest, restResponseChannel); + assertEquals("Mismatch on status", ResponseStatus.Ok, restResponseChannel.getStatus()); + + try { + accountService.getDatasetVersion(testAccount.getName(), testContainer.getName(), DATASET_NAME, version); + fail("Should fail due to dataset version has been deleted"); + } catch (AccountServiceException e) { + assertEquals("Mismatch on error code", AccountServiceErrorCode.Deleted, e.getErrorCode()); + } + + DatasetVersionRecord newDatasetVersionRecord = + accountService.getDatasetVersion(testAccount.getName(), testContainer.getName(), DATASET_NAME, newVersion); + assertEquals("Version mismatch", newVersion, newDatasetVersionRecord.getVersion()); + } + /** * Test the dataset version fallback when uploading named blob failed. */ diff --git a/ambry-test-utils/src/main/java/com/github/ambry/account/InMemAccountService.java b/ambry-test-utils/src/main/java/com/github/ambry/account/InMemAccountService.java index f1533119a1..1768c2772a 100644 --- a/ambry-test-utils/src/main/java/com/github/ambry/account/InMemAccountService.java +++ b/ambry-test-utils/src/main/java/com/github/ambry/account/InMemAccountService.java @@ -158,12 +158,30 @@ public synchronized DatasetVersionRecord addDatasetVersion(String accountName, S } } DatasetVersionRecord datasetVersionRecord = - new DatasetVersionRecord(accountId, containerId, datasetName, version, updatedExpirationTimeMs); + new DatasetVersionRecord(accountId, containerId, datasetName, version, updatedExpirationTimeMs, + creationTimeInMs, null); idToDatasetVersionMap.get(new Pair<>(accountId, containerId)) .put(new Pair<>(datasetName, version), datasetVersionRecord); return datasetVersionRecord; } + @Override + public void renameDatasetVersion(String accountName, String containerName, String datasetName, String sourceVersion, + String targetVersion) throws AccountServiceException { + DatasetVersionRecord datasetVersionRecord = + getDatasetVersion(accountName, containerName, datasetName, sourceVersion); + deleteDatasetVersion(accountName, containerName, datasetName, sourceVersion); + addDatasetVersion(accountName, containerName, datasetName, targetVersion, + (datasetVersionRecord.getExpirationTimeMs() - datasetVersionRecord.getCreationTimeMs()) / 1000, + datasetVersionRecord.getCreationTimeMs(), false, null); + Account account = nameToAccountMap.get(accountName); + short accountId = account.getId(); + short containerId = account.getContainerByName(containerName).getId(); + DatasetVersionRecord datasetVersionRecordToUpdate = + idToDatasetVersionMap.get(new Pair<>(accountId, containerId)).get(new Pair<>(datasetName, targetVersion)); + datasetVersionRecordToUpdate.setRenameFrom(sourceVersion); + } + @Override public synchronized DatasetVersionRecord getDatasetVersion(String accountName, String containerName, String datasetName, String version) throws AccountServiceException { @@ -187,6 +205,15 @@ public synchronized void deleteDatasetVersion(String accountName, String contain idToDatasetVersionMap.get(new Pair<>(accountId, containerId)).remove(new Pair<>(datasetName, version)); } + @Override + public synchronized void deleteDatasetVersionForDatasetDelete(String accountName, String containerName, + String datasetName, String version) throws AccountServiceException { + Account account = nameToAccountMap.get(accountName); + short accountId = account.getId(); + short containerId = account.getContainerByName(containerName).getId(); + idToDatasetVersionMap.get(new Pair<>(accountId, containerId)).remove(new Pair<>(datasetName, version)); + } + @Override public synchronized List getAllValidVersionsOutOfRetentionCount(String accountName, String containerName, String datasetName) throws AccountServiceException { @@ -324,7 +351,7 @@ public synchronized void updateDatasetVersionTtl(String accountName, String cont throw new AccountServiceException("Dataset version has been deleted", AccountServiceErrorCode.Deleted); } DatasetVersionRecord updatedDatasetVersionRecord = - new DatasetVersionRecord(accountId, containerId, datasetName, version, Utils.Infinite_Time); + new DatasetVersionRecord(accountId, containerId, datasetName, version, Utils.Infinite_Time, datasetVersionRecord.getRenameFrom()); idToDatasetVersionMap.get(new Pair<>(accountId, containerId)) .put(new Pair<>(datasetName, version), updatedDatasetVersionRecord); }