diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index c818c07b1ac..f153823db7c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -354,6 +355,24 @@ public V getValue() { public String toString() { return "(key=" + key + ", value=" + value + ")"; } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof KeyValue)) { + return false; + } + KeyValue kv = (KeyValue) obj; + try { + return getKey().equals(kv.getKey()) && getValue().equals(kv.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int hashCode() { + return Objects.hash(getKey(), getValue()); + } }; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 8fa8921cc9a..b70ea51fde5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -333,6 +333,7 @@ public static boolean isReadOnly( case DeleteSnapshot: case RenameSnapshot: case SnapshotMoveDeletedKeys: + case SnapshotMoveTableKeys: case SnapshotPurge: case RecoverLease: case SetTimes: diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java similarity index 52% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java index be4ea69095b..d77f9bf9d8d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.commons.compress.utils.Lists; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -32,20 +33,26 @@ import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; +import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; import org.junit.jupiter.api.Order; @@ -53,17 +60,27 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Random; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; @@ -72,6 +89,11 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; /** * Test Snapshot Deleting Service. @@ -80,10 +102,10 @@ @Timeout(300) @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(OrderAnnotation.class) -public class TestSnapshotDeletingService { +public class TestSnapshotDeletingServiceIntegrationTest { private static final Logger LOG = - LoggerFactory.getLogger(TestSnapshotDeletingService.class); + LoggerFactory.getLogger(TestSnapshotDeletingServiceIntegrationTest.class); private static boolean omRatisEnabled = true; private static final ByteBuffer CONTENT = ByteBuffer.allocate(1024 * 1024 * 16); @@ -147,7 +169,7 @@ public void testSnapshotSplitAndMove() throws Exception { Table snapshotInfoTable = om.getMetadataManager().getSnapshotInfoTable(); - createSnapshotDataForBucket1(); + createSnapshotDataForBucket(bucket1); assertTableRowCount(snapshotInfoTable, 2); GenericTestUtils.waitFor(() -> snapshotDeletingService @@ -174,7 +196,7 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { om.getMetadataManager().getSnapshotInfoTable(); runIndividualTest = false; - createSnapshotDataForBucket1(); + createSnapshotDataForBucket(bucket1); BucketArgs bucketArgs = new BucketArgs.Builder() .setBucketLayout(BucketLayout.LEGACY) @@ -454,6 +476,228 @@ public void testSnapshotWithFSO() throws Exception { rcSnap1.close(); } + private DirectoryDeletingService getMockedDirectoryDeletingService(AtomicBoolean dirDeletionWaitStarted, + AtomicBoolean dirDeletionStarted) + throws InterruptedException, TimeoutException { + OzoneManager ozoneManager = Mockito.spy(om); + om.getKeyManager().getDirDeletingService().shutdown(); + GenericTestUtils.waitFor(() -> om.getKeyManager().getDirDeletingService().getThreadCount() == 0, 1000, + 100000); + DirectoryDeletingService directoryDeletingService = Mockito.spy(new DirectoryDeletingService(10000, + TimeUnit.MILLISECONDS, 100000, ozoneManager, cluster.getConf())); + directoryDeletingService.shutdown(); + GenericTestUtils.waitFor(() -> directoryDeletingService.getThreadCount() == 0, 1000, + 100000); + when(ozoneManager.getMetadataManager()).thenAnswer(i -> { + // Wait for SDS to reach DDS wait block before processing any deleted directories. + GenericTestUtils.waitFor(dirDeletionWaitStarted::get, 1000, 100000); + dirDeletionStarted.set(true); + return i.callRealMethod(); + }); + return directoryDeletingService; + } + + private KeyDeletingService getMockedKeyDeletingService(AtomicBoolean keyDeletionWaitStarted, + AtomicBoolean keyDeletionStarted) + throws InterruptedException, TimeoutException, IOException { + OzoneManager ozoneManager = Mockito.spy(om); + om.getKeyManager().getDeletingService().shutdown(); + GenericTestUtils.waitFor(() -> om.getKeyManager().getDeletingService().getThreadCount() == 0, 1000, + 100000); + KeyManager keyManager = Mockito.spy(om.getKeyManager()); + when(ozoneManager.getKeyManager()).thenReturn(keyManager); + KeyDeletingService keyDeletingService = Mockito.spy(new KeyDeletingService(ozoneManager, + ozoneManager.getScmClient().getBlockClient(), keyManager, 10000, + 100000, cluster.getConf())); + keyDeletingService.shutdown(); + GenericTestUtils.waitFor(() -> keyDeletingService.getThreadCount() == 0, 1000, + 100000); + when(keyManager.getPendingDeletionKeys(anyInt())).thenAnswer(i -> { + // wait for SDS to reach the KDS wait block before processing any key. + GenericTestUtils.waitFor(keyDeletionWaitStarted::get, 1000, 100000); + keyDeletionStarted.set(true); + return i.callRealMethod(); + }); + return keyDeletingService; + } + + @SuppressWarnings("checkstyle:parameternumber") + private SnapshotDeletingService getMockedSnapshotDeletingService(KeyDeletingService keyDeletingService, + DirectoryDeletingService directoryDeletingService, + AtomicBoolean snapshotDeletionStarted, + AtomicBoolean keyDeletionWaitStarted, + AtomicBoolean dirDeletionWaitStarted, + AtomicBoolean keyDeletionStarted, + AtomicBoolean dirDeletionStarted, + OzoneBucket testBucket) + throws InterruptedException, TimeoutException, IOException { + OzoneManager ozoneManager = Mockito.spy(om); + om.getKeyManager().getSnapshotDeletingService().shutdown(); + GenericTestUtils.waitFor(() -> om.getKeyManager().getSnapshotDeletingService().getThreadCount() == 0, 1000, + 100000); + KeyManager keyManager = Mockito.spy(om.getKeyManager()); + OmMetadataManagerImpl omMetadataManager = Mockito.spy((OmMetadataManagerImpl)om.getMetadataManager()); + SnapshotChainManager unMockedSnapshotChainManager = + ((OmMetadataManagerImpl)om.getMetadataManager()).getSnapshotChainManager(); + SnapshotChainManager snapshotChainManager = Mockito.spy(unMockedSnapshotChainManager); + OmSnapshotManager omSnapshotManager = Mockito.spy(om.getOmSnapshotManager()); + when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); + when(ozoneManager.getKeyManager()).thenReturn(keyManager); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + when(omMetadataManager.getSnapshotChainManager()).thenReturn(snapshotChainManager); + when(keyManager.getDeletingService()).thenReturn(keyDeletingService); + when(keyManager.getDirDeletingService()).thenReturn(directoryDeletingService); + SnapshotDeletingService snapshotDeletingService = Mockito.spy(new SnapshotDeletingService(10000, + 100000, ozoneManager)); + snapshotDeletingService.shutdown(); + GenericTestUtils.waitFor(() -> snapshotDeletingService.getThreadCount() == 0, 1000, + 100000); + when(snapshotChainManager.iterator(anyBoolean())).thenAnswer(i -> { + Iterator itr = (Iterator) i.callRealMethod(); + return Lists.newArrayList(itr).stream().filter(uuid -> { + try { + SnapshotInfo snapshotInfo = SnapshotUtils.getSnapshotInfo(om, snapshotChainManager, uuid); + return snapshotInfo.getBucketName().equals(testBucket.getName()) && + snapshotInfo.getVolumeName().equals(testBucket.getVolumeName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).iterator(); + }); + when(snapshotChainManager.getLatestGlobalSnapshotId()) + .thenAnswer(i -> unMockedSnapshotChainManager.getLatestGlobalSnapshotId()); + when(snapshotChainManager.getOldestGlobalSnapshotId()) + .thenAnswer(i -> unMockedSnapshotChainManager.getOldestGlobalSnapshotId()); + doAnswer(i -> { + // KDS wait block reached in SDS. + GenericTestUtils.waitFor(() -> { + return keyDeletingService.isRunningOnAOS(); + }, 1000, 100000); + keyDeletionWaitStarted.set(true); + return i.callRealMethod(); + }).when(snapshotDeletingService).waitForKeyDeletingService(); + doAnswer(i -> { + // DDS wait block reached in SDS. + GenericTestUtils.waitFor(directoryDeletingService::isRunningOnAOS, 1000, 100000); + dirDeletionWaitStarted.set(true); + return i.callRealMethod(); + }).when(snapshotDeletingService).waitForDirDeletingService(); + doAnswer(i -> { + // Assert KDS & DDS is not running when SDS starts moving entries & assert all wait block, KDS processing + // AOS block & DDS AOS block have been executed. + Assertions.assertTrue(keyDeletionWaitStarted.get()); + Assertions.assertTrue(dirDeletionWaitStarted.get()); + Assertions.assertTrue(keyDeletionStarted.get()); + Assertions.assertTrue(dirDeletionStarted.get()); + Assertions.assertFalse(keyDeletingService.isRunningOnAOS()); + Assertions.assertFalse(directoryDeletingService.isRunningOnAOS()); + snapshotDeletionStarted.set(true); + return i.callRealMethod(); + }).when(omSnapshotManager).getSnapshot(anyString(), anyString(), anyString()); + return snapshotDeletingService; + } + + @Test + @Order(4) + public void testParallelExcecutionOfKeyDeletionAndSnapshotDeletion() throws Exception { + AtomicBoolean keyDeletionWaitStarted = new AtomicBoolean(false); + AtomicBoolean dirDeletionWaitStarted = new AtomicBoolean(false); + AtomicBoolean keyDeletionStarted = new AtomicBoolean(false); + AtomicBoolean dirDeletionStarted = new AtomicBoolean(false); + AtomicBoolean snapshotDeletionStarted = new AtomicBoolean(false); + Random random = new Random(); + String bucketName = "bucket" + random.nextInt(); + BucketArgs bucketArgs = new BucketArgs.Builder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build(); + OzoneBucket testBucket = TestDataUtil.createBucket( + client, VOLUME_NAME, bucketArgs, bucketName); + // mock keyDeletingService + KeyDeletingService keyDeletingService = getMockedKeyDeletingService(keyDeletionWaitStarted, keyDeletionStarted); + + // mock dirDeletingService + DirectoryDeletingService directoryDeletingService = getMockedDirectoryDeletingService(dirDeletionWaitStarted, + dirDeletionStarted); + + // mock snapshotDeletingService. + SnapshotDeletingService snapshotDeletingService = getMockedSnapshotDeletingService(keyDeletingService, + directoryDeletingService, snapshotDeletionStarted, keyDeletionWaitStarted, dirDeletionWaitStarted, + keyDeletionStarted, dirDeletionStarted, testBucket); + createSnapshotFSODataForBucket(testBucket); + List> renamesKeyEntries; + List>> deletedKeyEntries; + List> deletedDirEntries; + try (ReferenceCounted snapshot = om.getOmSnapshotManager().getSnapshot(testBucket.getVolumeName(), + testBucket.getName(), testBucket.getName() + "snap2")) { + renamesKeyEntries = snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000); + deletedKeyEntries = snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000); + deletedDirEntries = snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), + testBucket.getName(), 1000); + } + Thread keyDeletingThread = new Thread(() -> { + try { + keyDeletingService.runPeriodicalTaskNow(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Thread directoryDeletingThread = new Thread(() -> { + try { + directoryDeletingService.runPeriodicalTaskNow(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + ExecutorService snapshotDeletingThread = Executors.newFixedThreadPool(1); + Runnable snapshotDeletionRunnable = () -> { + try { + snapshotDeletingService.runPeriodicalTaskNow(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + keyDeletingThread.start(); + directoryDeletingThread.start(); + Future future = snapshotDeletingThread.submit(snapshotDeletionRunnable); + GenericTestUtils.waitFor(snapshotDeletionStarted::get, 1000, 30000); + future.get(); + try (ReferenceCounted snapshot = om.getOmSnapshotManager().getSnapshot(testBucket.getVolumeName(), + testBucket.getName(), testBucket.getName() + "snap2")) { + Assertions.assertEquals(Collections.emptyList(), + snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000)); + Assertions.assertEquals(Collections.emptyList(), + snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000)); + Assertions.assertEquals(Collections.emptyList(), + snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), + testBucket.getName(), 1000)); + } + List> aosRenamesKeyEntries = + om.getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000); + List>> aosDeletedKeyEntries = + om.getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), + testBucket.getName(), "", 1000); + List> aosDeletedDirEntries = + om.getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), + testBucket.getName(), 1000); + renamesKeyEntries.forEach(entry -> Assertions.assertTrue(aosRenamesKeyEntries.contains(entry))); + deletedKeyEntries.forEach(entry -> Assertions.assertTrue(aosDeletedKeyEntries.contains(entry))); + deletedDirEntries.forEach(entry -> Assertions.assertTrue(aosDeletedDirEntries.contains(entry))); + Mockito.reset(snapshotDeletingService); + SnapshotInfo snap2 = SnapshotUtils.getSnapshotInfo(om, testBucket.getVolumeName(), + testBucket.getName(), testBucket.getName() + "snap2"); + Assertions.assertEquals(snap2.getSnapshotStatus(), SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + future = snapshotDeletingThread.submit(snapshotDeletionRunnable); + future.get(); + Assertions.assertThrows(IOException.class, () -> SnapshotUtils.getSnapshotInfo(om, testBucket.getVolumeName(), + testBucket.getName(), testBucket.getName() + "snap2")); + cluster.restartOzoneManager(); + } + /* Flow ---- @@ -472,7 +716,7 @@ public void testSnapshotWithFSO() throws Exception { create snapshot3 delete snapshot2 */ - private void createSnapshotDataForBucket1() throws Exception { + private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws Exception { Table snapshotInfoTable = om.getMetadataManager().getSnapshotInfoTable(); Table deletedTable = @@ -482,70 +726,147 @@ private void createSnapshotDataForBucket1() throws Exception { OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); - TestDataUtil.createKey(bucket1, "bucket1key0", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); - TestDataUtil.createKey(bucket1, "bucket1key1", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key1", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); assertTableRowCount(keyTable, 2); // Create Snapshot 1. - client.getProxy().createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1snap1"); + client.getProxy().createSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap1"); assertTableRowCount(snapshotInfoTable, 1); // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket1, "bucket1key0", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); - TestDataUtil.createKey(bucket1, "bucket1key2", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key2", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. - client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1key1", false); + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "key1", false); // Key 2 is deleted here, which will be reclaimed here as // it is not being referenced by previous snapshot. - client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1key2", false); - client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1key0", false); + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "key2", false); + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "key0", false); assertTableRowCount(keyTable, 0); // one copy of bucket1key0 should also be reclaimed as it not same // but original deleted key created during overwrite should not be deleted assertTableRowCount(deletedTable, 2); // Create Snapshot 2. - client.getProxy().createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1snap2"); + client.getProxy().createSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap2"); assertTableRowCount(snapshotInfoTable, 2); // Key 2 is removed from the active Db's // deletedTable when Snapshot 2 is taken. assertTableRowCount(deletedTable, 0); - TestDataUtil.createKey(bucket1, "bucket1key3", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key3", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); - TestDataUtil.createKey(bucket1, "bucket1key4", ReplicationFactor.THREE, + TestDataUtil.createKey(bucket, bucket.getName() + "key4", ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT); - client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1key4", false); + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "key4", false); assertTableRowCount(keyTable, 1); assertTableRowCount(deletedTable, 0); // Create Snapshot 3. - client.getProxy().createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1snap3"); + client.getProxy().createSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap3"); assertTableRowCount(snapshotInfoTable, 3); SnapshotInfo snapshotInfo = metadataManager.getSnapshotInfoTable() - .get("/vol1/bucket1/bucket1snap2"); + .get(String.format("/%s/%s/%ssnap2", bucket.getVolumeName(), bucket.getName(), bucket.getName())); // Delete Snapshot 2. - client.getProxy().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, - "bucket1snap2"); + client.getProxy().deleteSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap2"); assertTableRowCount(snapshotInfoTable, 2); - verifySnapshotChain(snapshotInfo, "/vol1/bucket1/bucket1snap3"); + verifySnapshotChain(snapshotInfo, String.format("/%s/%s/%ssnap3", bucket.getVolumeName(), bucket.getName(), + bucket.getName())); + } + + + /* + Flow + ---- + create dir0/key0 + create dir1/key1 + overwrite dir0/key0 + create dir2/key2 + create snap1 + rename dir1/key1 -> dir1/key10 + delete dir1/key10 + delete dir2 + create snap2 + delete snap2 + */ + private synchronized void createSnapshotFSODataForBucket(OzoneBucket bucket) throws Exception { + Table snapshotInfoTable = + om.getMetadataManager().getSnapshotInfoTable(); + Table deletedTable = + om.getMetadataManager().getDeletedTable(); + Table deletedDirTable = + om.getMetadataManager().getDeletedDirTable(); + Table keyTable = + om.getMetadataManager().getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); + Table dirTable = + om.getMetadataManager().getDirectoryTable(); + Table renameTable = om.getMetadataManager().getSnapshotRenamedTable(); + OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) + om.getMetadataManager(); + Map countMap = + metadataManager.listTables().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> { + try { + return (int)metadataManager.countRowsInTable(e.getValue()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + })); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationFactor.THREE, + ReplicationType.RATIS, CONTENT); + TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", ReplicationFactor.THREE, + ReplicationType.RATIS, CONTENT); + assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 2); + assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 2); + + // Overwrite bucket1key0, This is a newer version of the key which should + // reclaimed as this is a different version of the key. + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationFactor.THREE, + ReplicationType.RATIS, CONTENT); + TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", ReplicationFactor.THREE, + ReplicationType.RATIS, CONTENT); + assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 3); + assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 3); + assertTableRowCount(deletedTable, countMap.get(deletedTable.getName()) + 1); + // create snap1 + client.getProxy().createSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap1"); + bucket.renameKey("dir1/" + bucket.getName() + "key1", "dir1/" + bucket.getName() + "key10"); + bucket.renameKey("dir1/", "dir10/"); + assertTableRowCount(renameTable, countMap.get(renameTable.getName()) + 2); + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), + "dir10/" + bucket.getName() + "key10", false); + assertTableRowCount(deletedTable, countMap.get(deletedTable.getName()) + 1); + // Key 2 is deleted here, which will be reclaimed here as + // it is not being referenced by previous snapshot. + client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), "dir2", true); + assertTableRowCount(deletedDirTable, countMap.get(deletedDirTable.getName()) + 1); + client.getProxy().createSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap2"); + // Delete Snapshot 2. + client.getProxy().deleteSnapshot(bucket.getVolumeName(), bucket.getName(), + bucket.getName() + "snap2"); + assertTableRowCount(snapshotInfoTable, countMap.get(snapshotInfoTable.getName()) + 2); } + private void verifySnapshotChain(SnapshotInfo deletedSnapshot, String nextSnapshot) throws Exception { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index eefcfa7552c..126adbdc51e 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -153,6 +153,7 @@ enum Type { GetServerDefaults = 134; GetQuotaRepairStatus = 135; StartQuotaRepair = 136; + SnapshotMoveTableKeys = 137; } enum SafeMode { @@ -295,6 +296,7 @@ message OMRequest { optional ServerDefaultsRequest ServerDefaultsRequest = 132; optional GetQuotaRepairStatusRequest GetQuotaRepairStatusRequest = 133; optional StartQuotaRepairRequest StartQuotaRepairRequest = 134; + optional SnapshotMoveTableKeysRequest SnapshotMoveTableKeysRequest = 135; } message OMResponse { @@ -1981,6 +1983,13 @@ message SnapshotMoveDeletedKeysRequest { repeated string deletedDirsToMove = 5; } +message SnapshotMoveTableKeysRequest { + optional hadoop.hdds.UUID fromSnapshotID = 1; + repeated SnapshotMoveKeyInfos deletedKeys = 2; + repeated SnapshotMoveKeyInfos deletedDirs = 3; + repeated hadoop.hdds.KeyValue renamedKeys = 4; +} + message SnapshotMoveKeyInfos { optional string key = 1; repeated KeyInfo keyInfos = 2; diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index cf0819ca527..67f7ce2f07c 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -116,6 +116,22 @@ public interface OMMetadataManager extends DBStoreHAManager { */ String getBucketKey(String volume, String bucket); + /** + * Given a volume and bucket, return the corresponding DB key prefix. + * + * @param volume - Volume name + * @param bucket - Bucket name + */ + String getBucketKeyPrefix(String volume, String bucket); + + /** + * Given a volume and bucket, return the corresponding DB key prefix for FSO buckets. + * + * @param volume - Volume name + * @param bucket - Bucket name + */ + String getBucketKeyPrefixFSO(String volume, String bucket) throws IOException; + /** * Given a volume, bucket and a key, return the corresponding DB key. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index b7fa5d746fb..068ba9aa4ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -18,6 +18,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -28,6 +29,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; @@ -35,6 +37,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.List; /** @@ -119,6 +122,29 @@ ListKeysResult listKeys(String volumeName, String bucketName, String startKey, */ PendingKeysDeletion getPendingDeletionKeys(int count) throws IOException; + /** + * Returns a list rename entries from the snapshotRenamedTable. + * + * @param size max number of keys to return. + * @return a Pair of list of {@link org.apache.hadoop.hdds.utils.db.Table.KeyValue} representing the keys in the + * underlying metadataManager. + * @throws IOException + */ + List> getRenamesKeyEntries( + String volume, String bucket, String startKey, int size) throws IOException; + + + /** + * Returns a list deleted entries from the deletedTable. + * + * @param size max number of keys to return. + * @return a Pair of list of {@link org.apache.hadoop.hdds.utils.db.Table.KeyValue} representing the keys in the + * underlying metadataManager. + * @throws IOException + */ + List>> getDeletedKeyEntries( + String volume, String bucket, String startKey, int size) throws IOException; + /** * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. @@ -216,6 +242,26 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, */ Table.KeyValue getPendingDeletionDir() throws IOException; + /** + * Returns an iterator for pending deleted directories. + * @throws IOException + */ + TableIterator> getDeletedDirEntries( + String volume, String bucket) throws IOException; + + default List> getDeletedDirEntries(String volume, String bucket, int size) + throws IOException { + List> deletedDirEntries = new ArrayList<>(size); + try (TableIterator> iterator = + getDeletedDirEntries(volume, bucket)) { + while (deletedDirEntries.size() < size && iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + deletedDirEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue())); + } + return deletedDirEntries; + } + } + /** * Returns all sub directories under the given parent directory. * @@ -243,7 +289,7 @@ List getPendingDeletionSubFiles(long volumeId, * Returns the instance of Directory Deleting Service. * @return Background service. */ - BackgroundService getDirDeletingService(); + DirectoryDeletingService getDirDeletingService(); /** * Returns the instance of Open Key Cleanup Service. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 6d276d95284..e3f56f2deaf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -37,6 +37,7 @@ import java.util.Stack; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -86,6 +87,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; @@ -189,7 +191,7 @@ public class KeyManagerImpl implements KeyManager { private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; - private BackgroundService dirDeletingService; + private DirectoryDeletingService dirDeletingService; private final OMPerformanceMetrics metrics; private BackgroundService openKeyCleanupService; @@ -305,7 +307,7 @@ public void start(OzoneConfiguration configuration) { try { snapshotDeletingService = new SnapshotDeletingService( snapshotServiceInterval, snapshotServiceTimeout, - ozoneManager, scmClient.getBlockClient()); + ozoneManager); snapshotDeletingService.start(); } catch (IOException e) { LOG.error("Error starting Snapshot Deleting Service", e); @@ -662,6 +664,60 @@ public PendingKeysDeletion getPendingDeletionKeys(final int count) .getPendingDeletionKeys(count, ozoneManager.getOmSnapshotManager()); } + private List> getTableEntries(String startKey, + TableIterator> tableIterator, + Function valueFunction, int size) throws IOException { + List> entries = new ArrayList<>(); + /* Seek to the start key if it not null. The next key in queue is ensured to start with the bucket + prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. + */ + if (startKey != null) { + tableIterator.seek(startKey); + tableIterator.seekToFirst(); + } + int currentCount = 0; + while (tableIterator.hasNext() && currentCount < size) { + Table.KeyValue kv = tableIterator.next(); + if (kv != null) { + entries.add(Table.newKeyValue(kv.getKey(), valueFunction.apply(kv.getValue()))); + currentCount++; + } + } + return entries; + } + + private Optional getBucketPrefix(String volumeName, String bucketName, boolean isFSO) throws IOException { + // Bucket prefix would be empty if both volume & bucket is empty i.e. either null or "". + if (StringUtils.isEmpty(volumeName) && StringUtils.isEmpty(bucketName)) { + return Optional.empty(); + } else if (StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(volumeName)) { + throw new IOException("One of volume : " + volumeName + ", bucket: " + bucketName + " is empty." + + " Either both should be empty or none of the arguments should be empty"); + } + return isFSO ? Optional.of(metadataManager.getBucketKeyPrefixFSO(volumeName, bucketName)) : + Optional.of(metadataManager.getBucketKeyPrefix(volumeName, bucketName)); + } + + @Override + public List> getRenamesKeyEntries( + String volume, String bucket, String startKey, int size) throws IOException { + Optional bucketPrefix = getBucketPrefix(volume, bucket, false); + try (TableIterator> + renamedKeyIter = metadataManager.getSnapshotRenamedTable().iterator(bucketPrefix.orElse(""))) { + return getTableEntries(startKey, renamedKeyIter, Function.identity(), size); + } + } + + @Override + public List>> getDeletedKeyEntries( + String volume, String bucket, String startKey, int size) throws IOException { + Optional bucketPrefix = getBucketPrefix(volume, bucket, false); + try (TableIterator> + delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { + return getTableEntries(startKey, delKeyIter, RepeatedOmKeyInfo::cloneOmKeyInfoList, size); + } + } + @Override public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, int count, BucketLayout bucketLayout, Duration leaseThreshold) throws IOException { @@ -688,7 +744,7 @@ public KeyDeletingService getDeletingService() { } @Override - public BackgroundService getDirDeletingService() { + public DirectoryDeletingService getDirDeletingService() { return dirDeletingService; } @@ -723,8 +779,7 @@ public boolean isSstFilteringSvcEnabled() { TimeUnit.MILLISECONDS); return serviceInterval != DISABLE_VALUE; } - - + @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix) throws OMException { @@ -1325,7 +1380,6 @@ private OmKeyInfo createFakeDirIfShould(String volume, String bucket, return null; } - private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, String clientAddress, boolean skipFileNotFoundError) throws IOException { final String volumeName = args.getVolumeName(); @@ -1784,17 +1838,13 @@ private List buildFinalStatusList( } fileStatusFinalList.add(fileStatus); } - return sortPipelineInfo(fileStatusFinalList, keyInfoList, omKeyArgs, clientAddress); } - private List sortPipelineInfo( List fileStatusFinalList, List keyInfoList, OmKeyArgs omKeyArgs, String clientAddress) throws IOException { - - if (omKeyArgs.getLatestVersionLocation()) { slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0])); } @@ -1976,6 +2026,13 @@ public Table.KeyValue getPendingDeletionDir() return null; } + @Override + public TableIterator> getDeletedDirEntries( + String volume, String bucket) throws IOException { + Optional bucketPrefix = getBucketPrefix(volume, bucket, true); + return metadataManager.getDeletedDirTable().iterator(bucketPrefix.orElse("")); + } + @Override public List getPendingDeletionSubDirs(long volumeId, long bucketId, OmKeyInfo parentInfo, long numEntries) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ee92dbc2fde..2e85fef162c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -824,7 +824,7 @@ public String getUserKey(String user) { /** * Given a volume and bucket, return the corresponding DB key. * - * @param volume - User name + * @param volume - Volume name * @param bucket - Bucket name */ @Override @@ -838,6 +838,22 @@ public String getBucketKey(String volume, String bucket) { return builder.toString(); } + /** + * {@inheritDoc} + */ + @Override + public String getBucketKeyPrefix(String volume, String bucket) { + return getOzoneKey(volume, bucket, OM_KEY_PREFIX); + } + + /** + * {@inheritDoc} + */ + @Override + public String getBucketKeyPrefixFSO(String volume, String bucket) throws IOException { + return getOzoneKeyFSO(volume, bucket, OM_KEY_PREFIX); + } + @Override public String getOzoneKey(String volume, String bucket, String key) { StringBuilder builder = new StringBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index b069a174cd0..e4102665d62 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -24,8 +24,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.NoSuchElementException; @@ -56,6 +58,7 @@ public class SnapshotChainManager { private final ConcurrentMap snapshotIdToTableKey; private UUID latestGlobalSnapshotId; private final boolean snapshotChainCorrupted; + private UUID oldestGlobalSnapshotId; public SnapshotChainManager(OMMetadataManager metadataManager) { globalSnapshotChain = Collections.synchronizedMap(new LinkedHashMap<>()); @@ -104,6 +107,8 @@ private void addSnapshotGlobal(UUID snapshotID, UUID prevGlobalID) // On add snapshot, set previous snapshot entry nextSnapshotID = // snapshotID globalSnapshotChain.get(prevGlobalID).setNextSnapshotId(snapshotID); + } else { + oldestGlobalSnapshotId = snapshotID; } globalSnapshotChain.put(snapshotID, @@ -171,7 +176,6 @@ private boolean deleteSnapshotGlobal(UUID snapshotID) throws IOException { // for node removal UUID next = globalSnapshotChain.get(snapshotID).getNextSnapshotId(); UUID prev = globalSnapshotChain.get(snapshotID).getPreviousSnapshotId(); - if (prev != null && !globalSnapshotChain.containsKey(prev)) { throw new IOException(String.format( "Global snapshot chain corruption. " + @@ -197,6 +201,9 @@ private boolean deleteSnapshotGlobal(UUID snapshotID) throws IOException { if (latestGlobalSnapshotId.equals(snapshotID)) { latestGlobalSnapshotId = prev; } + if (snapshotID.equals(oldestGlobalSnapshotId)) { + oldestGlobalSnapshotId = next; + } return true; } else { // snapshotID not found in snapshot chain, log warning and return @@ -382,6 +389,42 @@ public UUID getLatestGlobalSnapshotId() throws IOException { return latestGlobalSnapshotId; } + /** + * Get oldest of global snapshot in snapshot chain. + */ + public UUID getOldestGlobalSnapshotId() throws IOException { + validateSnapshotChain(); + return oldestGlobalSnapshotId; + } + + public Iterator iterator(final boolean reverse) throws IOException { + validateSnapshotChain(); + return new Iterator() { + private UUID currentSnapshotId = reverse ? getLatestGlobalSnapshotId() : getOldestGlobalSnapshotId(); + @Override + public boolean hasNext() { + return currentSnapshotId != null; + } + + @Override + public UUID next() { + try { + UUID prevSnapshotId = currentSnapshotId; + if (reverse && hasPreviousGlobalSnapshot(currentSnapshotId) || + !reverse && hasNextGlobalSnapshot(currentSnapshotId)) { + currentSnapshotId = + reverse ? previousGlobalSnapshot(currentSnapshotId) : nextGlobalSnapshot(currentSnapshotId); + } else { + currentSnapshotId = null; + } + return prevSnapshotId; + } catch (IOException e) { + throw new UncheckedIOException("Error while getting next snapshot for " + currentSnapshotId, e); + } + } + }; + } + /** * Get latest path snapshot in snapshot chain. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index ffaedaa06a9..5e324b376fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -78,6 +78,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveTableKeysRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotRenameRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; @@ -232,6 +233,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotRenameRequest(omRequest); case SnapshotMoveDeletedKeys: return new OMSnapshotMoveDeletedKeysRequest(omRequest); + case SnapshotMoveTableKeys: + return new OMSnapshotMoveTableKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); case SetSnapshotProperty: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 58fdb1232d3..2ddf308bb50 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -82,7 +82,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // Check the snapshot exists. SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot.getTableKey()); - nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); + nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); // Get next non-deleted snapshot. List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java new file mode 100644 index 00000000000..0eb0d3cd166 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveTableKeysResponse; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; +import org.apache.ratis.server.protocol.TermIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; + +/** + * Handles OMSnapshotMoveTableKeysRequest Request. + * This is an OM internal request. Does not need @RequireSnapshotFeatureState. + */ +public class OMSnapshotMoveTableKeysRequest extends OMClientRequest { + + private static final Logger LOG = LoggerFactory.getLogger(OMSnapshotMoveTableKeysRequest.class); + + public OMSnapshotMoveTableKeysRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + SnapshotMoveTableKeysRequest moveTableKeysRequest = getOmRequest().getSnapshotMoveTableKeysRequest(); + SnapshotInfo fromSnapshot = SnapshotUtils.getSnapshotInfo(ozoneManager, + snapshotChainManager, fromProtobuf(moveTableKeysRequest.getFromSnapshotID())); + String bucketKeyPrefix = omMetadataManager.getBucketKeyPrefix(fromSnapshot.getVolumeName(), + fromSnapshot.getBucketName()); + String bucketKeyPrefixFSO = omMetadataManager.getBucketKeyPrefixFSO(fromSnapshot.getVolumeName(), + fromSnapshot.getBucketName()); + + Set keys = new HashSet<>(); + List deletedKeys = new ArrayList<>(moveTableKeysRequest.getDeletedKeysList().size()); + + //validate deleted key starts with bucket prefix.[///] + for (SnapshotMoveKeyInfos deletedKey : moveTableKeysRequest.getDeletedKeysList()) { + // Filter only deleted keys with at least one keyInfo per key. + if (!deletedKey.getKeyInfosList().isEmpty()) { + deletedKeys.add(deletedKey); + if (!deletedKey.getKey().startsWith(bucketKeyPrefix)) { + throw new OMException("Deleted Key: " + deletedKey + " doesn't start with prefix " + bucketKeyPrefix, + OMException.ResultCodes.INVALID_KEY_NAME); + } + if (keys.contains(deletedKey.getKey())) { + throw new OMException("Duplicate Deleted Key: " + deletedKey + " in request", + OMException.ResultCodes.INVALID_REQUEST); + } else { + keys.add(deletedKey.getKey()); + } + } + } + + keys.clear(); + List renamedKeysList = new ArrayList<>(moveTableKeysRequest.getRenamedKeysList().size()); + //validate rename key starts with bucket prefix.[///] + for (HddsProtos.KeyValue renamedKey : moveTableKeysRequest.getRenamedKeysList()) { + if (renamedKey.hasKey() && renamedKey.hasValue()) { + renamedKeysList.add(renamedKey); + if (!renamedKey.getKey().startsWith(bucketKeyPrefix)) { + throw new OMException("Rename Key: " + renamedKey + " doesn't start with prefix " + bucketKeyPrefix, + OMException.ResultCodes.INVALID_KEY_NAME); + } + if (keys.contains(renamedKey.getKey())) { + throw new OMException("Duplicate rename Key: " + renamedKey + " in request", + OMException.ResultCodes.INVALID_REQUEST); + } else { + keys.add(renamedKey.getKey()); + } + } + } + keys.clear(); + + // Filter only deleted dirs with only one keyInfo per key. + List deletedDirs = new ArrayList<>(moveTableKeysRequest.getDeletedDirsList().size()); + //validate deleted key starts with bucket FSO path prefix.[///] + for (SnapshotMoveKeyInfos deletedDir : moveTableKeysRequest.getDeletedDirsList()) { + // Filter deleted directories with exactly one keyInfo per key. + if (deletedDir.getKeyInfosList().size() == 1) { + deletedDirs.add(deletedDir); + if (!deletedDir.getKey().startsWith(bucketKeyPrefixFSO)) { + throw new OMException("Deleted dir: " + deletedDir + " doesn't start with prefix " + + bucketKeyPrefixFSO, OMException.ResultCodes.INVALID_KEY_NAME); + } + if (keys.contains(deletedDir.getKey())) { + throw new OMException("Duplicate deleted dir Key: " + deletedDir + " in request", + OMException.ResultCodes.INVALID_REQUEST); + } else { + keys.add(deletedDir.getKey()); + } + } + } + return getOmRequest().toBuilder().setSnapshotMoveTableKeysRequest( + moveTableKeysRequest.toBuilder().clearDeletedDirs().clearDeletedKeys().clearRenamedKeys() + .addAllDeletedKeys(deletedKeys).addAllDeletedDirs(deletedDirs) + .addAllRenamedKeys(renamedKeysList).build()).build(); + } + + @Override + @DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT) + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + + SnapshotMoveTableKeysRequest moveTableKeysRequest = getOmRequest().getSnapshotMoveTableKeysRequest(); + + OMClientResponse omClientResponse; + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); + try { + SnapshotInfo fromSnapshot = SnapshotUtils.getSnapshotInfo(ozoneManager, + snapshotChainManager, fromProtobuf(moveTableKeysRequest.getFromSnapshotID())); + // If there is no snapshot in the chain after the current snapshot move the keys to Active Object Store. + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); + + // If next snapshot is not active then ignore move. Since this could be a redundant operations. + if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + throw new OMException("Next snapshot : " + nextSnapshot + " in chain is not active.", + OMException.ResultCodes.INVALID_SNAPSHOT_ERROR); + } + + // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. + fromSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshot)); + if (nextSnapshot != null) { + nextSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), nextSnapshot)); + } + omClientResponse = new OMSnapshotMoveTableKeysResponse(omResponse.build(), fromSnapshot, nextSnapshot, + moveTableKeysRequest.getDeletedKeysList(), moveTableKeysRequest.getDeletedDirsList(), + moveTableKeysRequest.getRenamedKeysList()); + } catch (IOException ex) { + omClientResponse = new OMSnapshotMoveTableKeysResponse(createErrorOMResponse(omResponse, ex)); + } + return omClientResponse; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 6602f52514b..ca29d4e112b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -105,7 +105,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } SnapshotInfo nextSnapshot = - SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); + SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); // Step 1: Update the deep clean flag for the next active snapshot updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index f39d5827a0c..7d1b7f237b2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -40,6 +40,7 @@ import java.util.List; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.createMergedRepeatedOmKeyInfoFromDeletedTableEntry; /** * Response for OMSnapshotMoveDeletedKeysRequest. @@ -199,8 +200,7 @@ private void processKeys(BatchOperation batchOp, } for (SnapshotMoveKeyInfos dBKey : nextDBKeysList) { - RepeatedOmKeyInfo omKeyInfos = - createRepeatedOmKeyInfo(dBKey, metadataManager); + RepeatedOmKeyInfo omKeyInfos = createMergedRepeatedOmKeyInfoFromDeletedTableEntry(dBKey, metadataManager); if (omKeyInfos == null) { continue; } @@ -223,36 +223,5 @@ public static RepeatedOmKeyInfo createRepeatedOmKeyInfo( return result; } - - private RepeatedOmKeyInfo createRepeatedOmKeyInfo( - SnapshotMoveKeyInfos snapshotMoveKeyInfos, - OMMetadataManager metadataManager) throws IOException { - String dbKey = snapshotMoveKeyInfos.getKey(); - List keyInfoList = snapshotMoveKeyInfos.getKeyInfosList(); - // When older version of keys are moved to the next snapshot's deletedTable - // The newer version might also be in the next snapshot's deletedTable and - // it might overwrite. This is to avoid that and also avoid having - // orphans blocks. - RepeatedOmKeyInfo result = metadataManager.getDeletedTable().get(dbKey); - - for (KeyInfo keyInfo : keyInfoList) { - OmKeyInfo omKeyInfo = OmKeyInfo.getFromProtobuf(keyInfo); - if (result == null) { - result = new RepeatedOmKeyInfo(omKeyInfo); - } else if (!isSameAsLatestOmKeyInfo(omKeyInfo, result)) { - result.addOmKeyInfo(omKeyInfo); - } - } - - return result; - } - - private boolean isSameAsLatestOmKeyInfo(OmKeyInfo omKeyInfo, - RepeatedOmKeyInfo result) { - int size = result.getOmKeyInfoList().size(); - assert size > 0; - OmKeyInfo keyInfoFromRepeated = result.getOmKeyInfoList().get(size - 1); - return omKeyInfo.equals(keyInfoFromRepeated); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java new file mode 100644 index 00000000000..b06570afb14 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import jakarta.annotation.Nonnull; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.createMergedRepeatedOmKeyInfoFromDeletedTableEntry; + +/** + * Response for OMSnapshotMoveDeletedKeysRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotMoveTableKeysResponse extends OMClientResponse { + + private SnapshotInfo fromSnapshot; + private SnapshotInfo nextSnapshot; + private List deletedKeys; + private List renameKeysList; + private List deletedDirs; + + public OMSnapshotMoveTableKeysResponse(OMResponse omResponse, + @Nonnull SnapshotInfo fromSnapshot, SnapshotInfo nextSnapshot, + List deletedKeys, + List deletedDirs, + List renamedKeys) { + super(omResponse); + this.fromSnapshot = fromSnapshot; + this.nextSnapshot = nextSnapshot; + this.deletedKeys = deletedKeys; + this.renameKeysList = renamedKeys; + this.deletedDirs = deletedDirs; + } + + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public OMSnapshotMoveTableKeysResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getOmSnapshotManager(); + + try (ReferenceCounted rcOmFromSnapshot = + omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) { + + OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get(); + + if (nextSnapshot != null) { + try (ReferenceCounted + rcOmNextSnapshot = omSnapshotManager.getSnapshot(nextSnapshot.getSnapshotId())) { + + OmSnapshot nextOmSnapshot = rcOmNextSnapshot.get(); + RDBStore nextSnapshotStore = (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); + // Init Batch Operation for snapshot db. + try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { + addKeysToNextSnapshot(writeBatch, nextOmSnapshot.getMetadataManager()); + nextSnapshotStore.commitBatchOperation(writeBatch); + nextSnapshotStore.getDb().flushWal(true); + nextSnapshotStore.getDb().flush(); + } + } + } else { + // Handle the case where there is no next Snapshot. + addKeysToNextSnapshot(batchOperation, omMetadataManager); + } + + // Update From Snapshot Deleted Table. + RDBStore fromSnapshotStore = (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = fromSnapshotStore.initBatchOperation()) { + deleteKeysFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); + fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); + fromSnapshotStore.getDb().flushWal(true); + fromSnapshotStore.getDb().flush(); + } + } + + // Flush snapshot info to rocksDB. + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); + if (nextSnapshot != null) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, nextSnapshot.getTableKey(), nextSnapshot); + } + } + + private void deleteKeysFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) + throws IOException { + for (SnapshotMoveKeyInfos deletedOmKeyInfo : deletedKeys) { + // Delete keys from current snapshot that are moved to next snapshot. + fromSnapshotMetadataManager.getDeletedTable().deleteWithBatch(batchOp, deletedOmKeyInfo.getKey()); + } + + // Delete rename keys from current snapshot that are moved to next snapshot. + for (HddsProtos.KeyValue renameEntry : renameKeysList) { + fromSnapshotMetadataManager.getSnapshotRenamedTable().deleteWithBatch(batchOp, renameEntry.getKey()); + } + + // Delete deletedDir from current snapshot that are moved to next snapshot. + for (SnapshotMoveKeyInfos deletedDirInfo : deletedDirs) { + fromSnapshotMetadataManager.getDeletedDirTable().deleteWithBatch(batchOp, deletedDirInfo.getKey()); + } + + } + + private void addKeysToNextSnapshot(BatchOperation batchOp, OMMetadataManager metadataManager) throws IOException { + + // Add renamed keys to the next snapshot or active DB. + for (HddsProtos.KeyValue renameEntry : renameKeysList) { + metadataManager.getSnapshotRenamedTable().putWithBatch(batchOp, renameEntry.getKey(), renameEntry.getValue()); + } + // Add deleted keys to the next snapshot or active DB. + for (SnapshotMoveKeyInfos deletedKeyInfo : deletedKeys) { + RepeatedOmKeyInfo omKeyInfos = createMergedRepeatedOmKeyInfoFromDeletedTableEntry(deletedKeyInfo, + metadataManager); + metadataManager.getDeletedTable().putWithBatch(batchOp, deletedKeyInfo.getKey(), omKeyInfos); + } + // Add deleted dir keys to the next snapshot or active DB. + for (SnapshotMoveKeyInfos deletedDirInfo : deletedDirs) { + metadataManager.getDeletedDirTable().putWithBatch(batchOp, deletedDirInfo.getKey(), + OmKeyInfo.getFromProtobuf(deletedDirInfo.getKeyInfosList().get(0))); + } + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index c8703c3c4c6..ad16c49d5e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -82,6 +82,7 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService { private final long pathLimitPerTask; private final int ratisByteLimit; private final AtomicBoolean suspended; + private AtomicBoolean isRunningOnAOS; public DirectoryDeletingService(long interval, TimeUnit unit, long serviceTimeout, OzoneManager ozoneManager, @@ -98,6 +99,7 @@ public DirectoryDeletingService(long interval, TimeUnit unit, // always go to 90% of max limit for request as other header will be added this.ratisByteLimit = (int) (limit * 0.9); this.suspended = new AtomicBoolean(false); + this.isRunningOnAOS = new AtomicBoolean(false); } private boolean shouldRun() { @@ -108,6 +110,10 @@ private boolean shouldRun() { return getOzoneManager().isLeaderReady() && !suspended.get(); } + public boolean isRunningOnAOS() { + return isRunningOnAOS.get(); + } + /** * Suspend the service. */ @@ -127,11 +133,16 @@ public void resume() { @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); - queue.add(new DirectoryDeletingService.DirDeletingTask()); + queue.add(new DirectoryDeletingService.DirDeletingTask(this)); return queue; } - private class DirDeletingTask implements BackgroundTask { + private final class DirDeletingTask implements BackgroundTask { + private final DirectoryDeletingService directoryDeletingService; + + private DirDeletingTask(DirectoryDeletingService service) { + this.directoryDeletingService = service; + } @Override public int getPriority() { @@ -144,6 +155,7 @@ public BackgroundTaskResult call() { if (LOG.isDebugEnabled()) { LOG.debug("Running DirectoryDeletingService"); } + isRunningOnAOS.set(true); getRunCount().incrementAndGet(); long dirNum = 0L; long subDirNum = 0L; @@ -210,8 +222,11 @@ public BackgroundTaskResult call() { LOG.error("Error while running delete directories and files " + "background task. Will retry at next run.", e); } + isRunningOnAOS.set(false); + synchronized (directoryDeletingService) { + this.directoryDeletingService.notify(); + } } - // place holder by returning empty results of this call back. return BackgroundTaskResult.EmptyTaskResult.newResult(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 5e622cb1701..e7553004edf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -92,6 +92,7 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final Map exclusiveReplicatedSizeMap; private final Set completedExclusiveSizeSet; private final Map snapshotSeekMap; + private AtomicBoolean isRunningOnAOS; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -111,6 +112,7 @@ public KeyDeletingService(OzoneManager ozoneManager, this.exclusiveReplicatedSizeMap = new HashMap<>(); this.completedExclusiveSizeSet = new HashSet<>(); this.snapshotSeekMap = new HashMap<>(); + this.isRunningOnAOS = new AtomicBoolean(false); } /** @@ -123,10 +125,14 @@ public AtomicLong getDeletedKeyCount() { return deletedKeyCount; } + public boolean isRunningOnAOS() { + return isRunningOnAOS.get(); + } + @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); - queue.add(new KeyDeletingTask()); + queue.add(new KeyDeletingTask(this)); return queue; } @@ -169,7 +175,12 @@ public void setKeyLimitPerTask(int keyLimitPerTask) { * the blocks info in its deletedBlockLog), it removes these keys from the * DB. */ - private class KeyDeletingTask implements BackgroundTask { + private final class KeyDeletingTask implements BackgroundTask { + private final KeyDeletingService deletingService; + + private KeyDeletingTask(KeyDeletingService service) { + this.deletingService = service; + } @Override public int getPriority() { @@ -183,7 +194,7 @@ public BackgroundTaskResult call() { if (shouldRun()) { final long run = getRunCount().incrementAndGet(); LOG.debug("Running KeyDeletingService {}", run); - + isRunningOnAOS.set(true); int delCount = 0; try { // TODO: [SNAPSHOT] HDDS-7968. Reclaim eligible key blocks in @@ -217,6 +228,11 @@ public BackgroundTaskResult call() { } } + isRunningOnAOS.set(false); + synchronized (deletingService) { + this.deletingService.notify(); + } + // By design, no one cares about the results of this call back. return EmptyTaskResult.newResult(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index f85bd781b05..edc6c7a1629 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -20,54 +20,49 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.hdds.HddsUtils.toProtobuf; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; @@ -93,16 +88,17 @@ public class SnapshotDeletingService extends AbstractKeyDeletingService { private final AtomicBoolean suspended; private final OzoneConfiguration conf; private final AtomicLong successRunCount; - private final long snapshotDeletionPerTask; - private final int keyLimitPerSnapshot; + private final int keyLimitPerTask; + private final int snapshotDeletionPerTask; private final int ratisByteLimit; + private final long serviceTimeout; public SnapshotDeletingService(long interval, long serviceTimeout, - OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient) + OzoneManager ozoneManager) throws IOException { super(SnapshotDeletingService.class.getSimpleName(), interval, TimeUnit.MILLISECONDS, SNAPSHOT_DELETING_CORE_POOL_SIZE, - serviceTimeout, ozoneManager, scmClient); + serviceTimeout, ozoneManager, null); this.ozoneManager = ozoneManager; this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) @@ -111,8 +107,7 @@ public SnapshotDeletingService(long interval, long serviceTimeout, this.successRunCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); this.conf = ozoneManager.getConfiguration(); - this.snapshotDeletionPerTask = conf - .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, + this.snapshotDeletionPerTask = conf.getInt(SNAPSHOT_DELETING_LIMIT_PER_TASK, SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); int limit = (int) conf.getStorageSize( OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, @@ -120,9 +115,35 @@ public SnapshotDeletingService(long interval, long serviceTimeout, StorageUnit.BYTES); // always go to 90% of max limit for request as other header will be added this.ratisByteLimit = (int) (limit * 0.9); - this.keyLimitPerSnapshot = conf.getInt( + this.keyLimitPerTask = conf.getInt( OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); + this.serviceTimeout = serviceTimeout; + } + + // Wait for a notification from KeyDeletingService if the key deletion is running. This is to ensure, merging of + // entries do not start while the AOS is still processing the deleted keys. + @VisibleForTesting + public void waitForKeyDeletingService() throws InterruptedException { + KeyDeletingService keyDeletingService = getOzoneManager().getKeyManager().getDeletingService(); + synchronized (keyDeletingService) { + while (keyDeletingService.isRunningOnAOS()) { + keyDeletingService.wait(serviceTimeout); + } + } + } + + // Wait for a notification from DirectoryDeletingService if the directory deletion is running. This is to ensure, + // merging of entries do not start while the AOS is still processing the deleted keys. + @VisibleForTesting + public void waitForDirDeletingService() throws InterruptedException { + DirectoryDeletingService directoryDeletingService = getOzoneManager().getKeyManager() + .getDirDeletingService(); + synchronized (directoryDeletingService) { + while (directoryDeletingService.isRunningOnAOS()) { + directoryDeletingService.wait(serviceTimeout); + } + } } private class SnapshotDeletingTask implements BackgroundTask { @@ -136,316 +157,89 @@ public BackgroundTaskResult call() throws InterruptedException { getRunCount().incrementAndGet(); - ReferenceCounted rcOmSnapshot = null; - ReferenceCounted rcOmPreviousSnapshot = null; - - Table snapshotInfoTable = - ozoneManager.getMetadataManager().getSnapshotInfoTable(); - List purgeSnapshotKeys = new ArrayList<>(); - try (TableIterator> iterator = snapshotInfoTable.iterator()) { - + try { + int remaining = keyLimitPerTask; + Iterator iterator = chainManager.iterator(true); + List snapshotsToBePurged = new ArrayList<>(); long snapshotLimit = snapshotDeletionPerTask; - - while (iterator.hasNext() && snapshotLimit > 0) { - SnapshotInfo snapInfo = iterator.next().getValue(); - - // Only Iterate in deleted snapshot + while (iterator.hasNext() && snapshotLimit > 0 && remaining > 0) { + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, chainManager, iterator.next()); if (shouldIgnoreSnapshot(snapInfo)) { continue; } - - // Note: Can refactor this to use try-with-resources. - // Handling RC decrements manually for now to minimize conflicts. - rcOmSnapshot = omSnapshotManager.getSnapshot( - snapInfo.getVolumeName(), - snapInfo.getBucketName(), - snapInfo.getName()); - OmSnapshot omSnapshot = rcOmSnapshot.get(); - - Table snapshotDeletedTable = - omSnapshot.getMetadataManager().getDeletedTable(); - Table snapshotDeletedDirTable = - omSnapshot.getMetadataManager().getDeletedDirTable(); - - Table renamedTable = - omSnapshot.getMetadataManager().getSnapshotRenamedTable(); - - long volumeId = ozoneManager.getMetadataManager() - .getVolumeId(snapInfo.getVolumeName()); - // Get bucketInfo for the snapshot bucket to get bucket layout. - String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey( - snapInfo.getVolumeName(), snapInfo.getBucketName()); - OmBucketInfo bucketInfo = ozoneManager.getMetadataManager() - .getBucketTable().get(dbBucketKey); - - if (bucketInfo == null) { - // Decrement ref count - rcOmSnapshot.close(); - rcOmSnapshot = null; - throw new IllegalStateException("Bucket " + "/" + - snapInfo.getVolumeName() + "/" + snapInfo.getBucketName() + - " is not found. BucketInfo should not be null for snapshotted" + - " bucket. The OM is in unexpected state."); - } - - String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - String dbBucketKeyForDir = ozoneManager.getMetadataManager() - .getBucketKey(Long.toString(volumeId), - Long.toString(bucketInfo.getObjectID())) + OM_KEY_PREFIX; - - if (isSnapshotReclaimable(snapshotDeletedTable, - snapshotDeletedDirTable, snapshotBucketKey, dbBucketKeyForDir)) { - purgeSnapshotKeys.add(snapInfo.getTableKey()); - // Decrement ref count - rcOmSnapshot.close(); - rcOmSnapshot = null; + LOG.info("Started Snapshot Deletion Processing for snapshot : {}", snapInfo.getTableKey()); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, chainManager, snapInfo); + // Continue if the next snapshot is not active. This is to avoid unnecessary copies from one snapshot to + // another. + if (nextSnapshot != null && + nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { continue; } - //TODO: [SNAPSHOT] Add lock to deletedTable and Active DB. - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(snapInfo, chainManager); - Table previousKeyTable = null; - Table previousDirTable = null; - OmSnapshot omPreviousSnapshot = null; - - // Split RepeatedOmKeyInfo and update current snapshot deletedKeyTable - // and next snapshot deletedKeyTable. - if (previousSnapshot != null) { - rcOmPreviousSnapshot = omSnapshotManager.getSnapshot( - previousSnapshot.getVolumeName(), - previousSnapshot.getBucketName(), - previousSnapshot.getName()); - omPreviousSnapshot = rcOmPreviousSnapshot.get(); - - previousKeyTable = omPreviousSnapshot - .getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); - previousDirTable = omPreviousSnapshot - .getMetadataManager().getDirectoryTable(); + // nextSnapshot = null means entries would be moved to AOS. + if (nextSnapshot == null) { + waitForKeyDeletingService(); + waitForDirDeletingService(); } - - // Move key to either next non deleted snapshot's deletedTable - // or keep it in current snapshot deleted table. - List toReclaimList = new ArrayList<>(); - List toNextDBList = new ArrayList<>(); - // A list of renamed keys/files/dirs - List renamedList = new ArrayList<>(); - List dirsToMove = new ArrayList<>(); - - long remainNum = handleDirectoryCleanUp(snapshotDeletedDirTable, - previousDirTable, renamedTable, dbBucketKeyForDir, snapInfo, - omSnapshot, dirsToMove, renamedList); - int deletionCount = 0; - - try (TableIterator> deletedIterator = snapshotDeletedTable - .iterator()) { - - List keysToPurge = new ArrayList<>(); - deletedIterator.seek(snapshotBucketKey); - - while (deletedIterator.hasNext() && - deletionCount < remainNum) { - Table.KeyValue - deletedKeyValue = deletedIterator.next(); - String deletedKey = deletedKeyValue.getKey(); - - // Exit if it is out of the bucket scope. - if (!deletedKey.startsWith(snapshotBucketKey)) { - // If snapshot deletedKeyTable doesn't have any - // entry in the snapshot scope it can be reclaimed - break; - } - - RepeatedOmKeyInfo repeatedOmKeyInfo = deletedKeyValue.getValue(); - - SnapshotMoveKeyInfos.Builder toReclaim = SnapshotMoveKeyInfos - .newBuilder() - .setKey(deletedKey); - SnapshotMoveKeyInfos.Builder toNextDb = SnapshotMoveKeyInfos - .newBuilder() - .setKey(deletedKey); - HddsProtos.KeyValue.Builder renamedKey = HddsProtos.KeyValue - .newBuilder(); - - for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - splitRepeatedOmKeyInfo(toReclaim, toNextDb, renamedKey, - keyInfo, previousKeyTable, renamedTable, - bucketInfo, volumeId); + try (ReferenceCounted snapshot = omSnapshotManager.getSnapshot( + snapInfo.getVolumeName(), snapInfo.getBucketName(), snapInfo.getName())) { + KeyManager snapshotKeyManager = snapshot.get().getKeyManager(); + int moveCount = 0; + // Get all entries from deletedKeyTable. + List>> deletedKeyEntries = + snapshotKeyManager.getDeletedKeyEntries(snapInfo.getVolumeName(), snapInfo.getBucketName(), + null, remaining); + moveCount += deletedKeyEntries.size(); + // Get all entries from deletedDirTable. + List> deletedDirEntries = snapshotKeyManager.getDeletedDirEntries( + snapInfo.getVolumeName(), snapInfo.getBucketName(), remaining - moveCount); + moveCount += deletedDirEntries.size(); + // Get all entries from snapshotRenamedTable. + List> renameEntries = snapshotKeyManager.getRenamesKeyEntries( + snapInfo.getVolumeName(), snapInfo.getBucketName(), null, remaining - moveCount); + moveCount += renameEntries.size(); + if (moveCount > 0) { + List deletedKeys = new ArrayList<>(deletedKeyEntries.size()); + List deletedDirs = new ArrayList<>(deletedDirEntries.size()); + List renameKeys = new ArrayList<>(renameEntries.size()); + + // Convert deletedKeyEntries to SnapshotMoveKeyInfos. + for (Table.KeyValue> deletedEntry : deletedKeyEntries) { + deletedKeys.add(SnapshotMoveKeyInfos.newBuilder().setKey(deletedEntry.getKey()) + .addAllKeyInfos(deletedEntry.getValue() + .stream().map(val -> val.getProtobuf(ClientVersion.CURRENT_VERSION)) + .collect(Collectors.toList())).build()); } - // If all the KeyInfos are reclaimable in RepeatedOmKeyInfo - // then no need to update current snapshot deletedKeyTable. - if (!(toReclaim.getKeyInfosCount() == - repeatedOmKeyInfo.getOmKeyInfoList().size())) { - toReclaimList.add(toReclaim.build()); - toNextDBList.add(toNextDb.build()); - } else { - // The key can be reclaimed here. - List blocksForKeyDelete = omSnapshot - .getMetadataManager() - .getBlocksForKeyDelete(deletedKey); - if (blocksForKeyDelete != null) { - keysToPurge.addAll(blocksForKeyDelete); - } + // Convert deletedDirEntries to SnapshotMoveKeyInfos. + for (Table.KeyValue deletedDirEntry : deletedDirEntries) { + deletedDirs.add(SnapshotMoveKeyInfos.newBuilder().setKey(deletedDirEntry.getKey()) + .addKeyInfos(deletedDirEntry.getValue().getProtobuf(ClientVersion.CURRENT_VERSION)).build()); } - if (renamedKey.hasKey() && renamedKey.hasValue()) { - renamedList.add(renamedKey.build()); + // Convert renamedEntries to KeyValue. + for (Table.KeyValue renameEntry : renameEntries) { + renameKeys.add(HddsProtos.KeyValue.newBuilder().setKey(renameEntry.getKey()) + .setValue(renameEntry.getValue()).build()); } - deletionCount++; + submitSnapshotMoveDeletedKeys(snapInfo, deletedKeys, renameKeys, deletedDirs); + remaining -= moveCount; + } else { + snapshotsToBePurged.add(snapInfo.getTableKey()); } - - // Delete keys From deletedTable - processKeyDeletes(keysToPurge, omSnapshot.getKeyManager(), - null, snapInfo.getTableKey()); - successRunCount.incrementAndGet(); - } catch (IOException ex) { - LOG.error("Error while running Snapshot Deleting Service for " + - "snapshot " + snapInfo.getTableKey() + " with snapshotId " + - snapInfo.getSnapshotId() + ". Processed " + deletionCount + - " keys and " + (keyLimitPerSnapshot - remainNum) + - " directories and files", ex); } + successRunCount.incrementAndGet(); snapshotLimit--; - // Submit Move request to OM. - submitSnapshotMoveDeletedKeys(snapInfo, toReclaimList, - toNextDBList, renamedList, dirsToMove); - - // Properly decrement ref count for rcOmPreviousSnapshot - if (rcOmPreviousSnapshot != null) { - rcOmPreviousSnapshot.close(); - rcOmPreviousSnapshot = null; - } + } + if (!snapshotsToBePurged.isEmpty()) { + submitSnapshotPurgeRequest(snapshotsToBePurged); } } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); - } finally { - // Decrement ref counts - if (rcOmPreviousSnapshot != null) { - rcOmPreviousSnapshot.close(); - } - if (rcOmSnapshot != null) { - rcOmSnapshot.close(); - } } - submitSnapshotPurgeRequest(purgeSnapshotKeys); - return BackgroundTaskResult.EmptyTaskResult.newResult(); } - private boolean isSnapshotReclaimable( - Table snapshotDeletedTable, - Table snapshotDeletedDirTable, - String snapshotBucketKey, String dbBucketKeyForDir) throws IOException { - - boolean isDirTableCleanedUp = false; - boolean isKeyTableCleanedUp = false; - try (TableIterator> iterator = snapshotDeletedTable.iterator();) { - iterator.seek(snapshotBucketKey); - // If the next entry doesn't start with snapshotBucketKey then - // deletedKeyTable is already cleaned up. - isKeyTableCleanedUp = !iterator.hasNext() || !iterator.next().getKey() - .startsWith(snapshotBucketKey); - } - - try (TableIterator> - iterator = snapshotDeletedDirTable.iterator()) { - iterator.seek(dbBucketKeyForDir); - // If the next entry doesn't start with dbBucketKeyForDir then - // deletedDirTable is already cleaned up. - isDirTableCleanedUp = !iterator.hasNext() || !iterator.next().getKey() - .startsWith(dbBucketKeyForDir); - } - - return (isDirTableCleanedUp || snapshotDeletedDirTable.isEmpty()) && - (isKeyTableCleanedUp || snapshotDeletedTable.isEmpty()); - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private long handleDirectoryCleanUp( - Table snapshotDeletedDirTable, - Table previousDirTable, - Table renamedTable, - String dbBucketKeyForDir, SnapshotInfo snapInfo, - OmSnapshot omSnapshot, List dirsToMove, - List renamedList) { - - long dirNum = 0L; - long subDirNum = 0L; - long subFileNum = 0L; - long remainNum = keyLimitPerSnapshot; - int consumedSize = 0; - List purgePathRequestList = new ArrayList<>(); - List> allSubDirList - = new ArrayList<>(keyLimitPerSnapshot); - try (TableIterator> deletedDirIterator = - snapshotDeletedDirTable.iterator()) { - - long startTime = Time.monotonicNow(); - deletedDirIterator.seek(dbBucketKeyForDir); - - while (deletedDirIterator.hasNext()) { - Table.KeyValue deletedDir = - deletedDirIterator.next(); - String deletedDirKey = deletedDir.getKey(); - - // Exit for dirs out of snapshot scope. - if (!deletedDirKey.startsWith(dbBucketKeyForDir)) { - break; - } - - if (isDirReclaimable(deletedDir, previousDirTable, - renamedTable, renamedList)) { - // Reclaim here - PurgePathRequest request = prepareDeleteDirRequest( - remainNum, deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, omSnapshot.getKeyManager()); - if (isBufferLimitCrossed(ratisByteLimit, consumedSize, - request.getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum = 0; - break; - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum = MIN_ERR_LIMIT_PER_TASK; - request = prepareDeleteDirRequest( - remainNum, deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, omSnapshot.getKeyManager()); - } - consumedSize += request.getSerializedSize(); - purgePathRequestList.add(request); - remainNum = remainNum - request.getDeletedSubFilesCount(); - remainNum = remainNum - request.getMarkDeletedSubDirsCount(); - // Count up the purgeDeletedDir, subDirs and subFiles - if (request.getDeletedDir() != null - && !request.getDeletedDir().isEmpty()) { - dirNum++; - } - subDirNum += request.getMarkDeletedSubDirsCount(); - subFileNum += request.getDeletedSubFilesCount(); - } else { - dirsToMove.add(deletedDir.getKey()); - } - } - - remainNum = optimizeDirDeletesAndSubmitRequest(remainNum, dirNum, - subDirNum, subFileNum, allSubDirList, purgePathRequestList, - snapInfo.getTableKey(), startTime, ratisByteLimit - consumedSize, - omSnapshot.getKeyManager()); - } catch (IOException e) { - LOG.error("Error while running delete directories and files for " + - "snapshot " + snapInfo.getTableKey() + " in snapshot deleting " + - "background task. Will retry at next run.", e); - } - - return remainNum; - } - private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { if (!purgeSnapshotKeys.isEmpty()) { SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest @@ -463,92 +257,36 @@ private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { } } - @SuppressWarnings("checkstyle:ParameterNumber") - private void splitRepeatedOmKeyInfo(SnapshotMoveKeyInfos.Builder toReclaim, - SnapshotMoveKeyInfos.Builder toNextDb, - HddsProtos.KeyValue.Builder renamedKey, OmKeyInfo keyInfo, - Table previousKeyTable, - Table renamedTable, - OmBucketInfo bucketInfo, long volumeId) throws IOException { - - if (isKeyReclaimable(previousKeyTable, renamedTable, - keyInfo, bucketInfo, volumeId, renamedKey)) { - // Update in current db's deletedKeyTable - toReclaim.addKeyInfos(keyInfo - .getProtobuf(ClientVersion.CURRENT_VERSION)); - } else { - // Move to next non deleted snapshot's deleted table - toNextDb.addKeyInfos(keyInfo.getProtobuf( - ClientVersion.CURRENT_VERSION)); - } - } - - private boolean isDirReclaimable( - Table.KeyValue deletedDir, - Table previousDirTable, - Table renamedTable, - List renamedList) throws IOException { - - if (previousDirTable == null) { - return true; - } - - String deletedDirDbKey = deletedDir.getKey(); - OmKeyInfo deletedDirInfo = deletedDir.getValue(); - String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( - deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), - deletedDirInfo.getObjectID()); - - /* - snapshotRenamedTable: /volumeName/bucketName/objectID -> - /volumeId/bucketId/parentId/dirName - */ - String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey); - String prevDbKey = null; - - if (dbKeyBeforeRename != null) { - prevDbKey = dbKeyBeforeRename; - HddsProtos.KeyValue renamedDir = HddsProtos.KeyValue - .newBuilder() - .setKey(dbRenameKey) - .setValue(dbKeyBeforeRename) - .build(); - renamedList.add(renamedDir); - } else { - // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to - // OzoneDeletePathKey. Changing it back to check the previous DirTable. - prevDbKey = ozoneManager.getMetadataManager() - .getOzoneDeletePathDirKey(deletedDirDbKey); - } - - OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey); - if (prevDirectoryInfo == null) { - return true; - } - - return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); - } - - public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - List toReclaimList, - List toNextDBList, - List renamedList, - List dirsToMove) throws InterruptedException { + private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, + List deletedKeys, + List renamedList, + List dirsToMove) { - SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = - SnapshotMoveDeletedKeysRequest.newBuilder() - .setFromSnapshot(snapInfo.getProtobuf()); + SnapshotMoveTableKeysRequest.Builder moveDeletedKeysBuilder = SnapshotMoveTableKeysRequest.newBuilder() + .setFromSnapshotID(toProtobuf(snapInfo.getSnapshotId())); - SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder - .addAllReclaimKeys(toReclaimList) - .addAllNextDBKeys(toNextDBList) + SnapshotMoveTableKeysRequest moveDeletedKeys = moveDeletedKeysBuilder + .addAllDeletedKeys(deletedKeys) .addAllRenamedKeys(renamedList) - .addAllDeletedDirsToMove(dirsToMove) + .addAllDeletedDirs(dirsToMove) .build(); + if (isBufferLimitCrossed(ratisByteLimit, 0, moveDeletedKeys.getSerializedSize())) { + int remaining = MIN_ERR_LIMIT_PER_TASK; + deletedKeys = deletedKeys.subList(0, Math.min(remaining, deletedKeys.size())); + remaining -= deletedKeys.size(); + renamedList = renamedList.subList(0, Math.min(remaining, renamedList.size())); + remaining -= renamedList.size(); + dirsToMove = dirsToMove.subList(0, Math.min(remaining, dirsToMove.size())); + moveDeletedKeys = moveDeletedKeysBuilder + .addAllDeletedKeys(deletedKeys) + .addAllRenamedKeys(renamedList) + .addAllDeletedDirs(dirsToMove) + .build(); + } OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SnapshotMoveDeletedKeys) - .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) + .setCmdType(Type.SnapshotMoveTableKeys) + .setSnapshotMoveTableKeysRequest(moveDeletedKeys) .setClientId(clientId.toString()) .build(); @@ -557,20 +295,26 @@ public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, } } - public void submitRequest(OMRequest omRequest) { + private void submitRequest(OMRequest omRequest) { try { OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, getRunCount().get()); } catch (ServiceException e) { - LOG.error("Snapshot Deleting request failed. " + - "Will retry at next run.", e); + LOG.error("Request: {} fired by SnapshotDeletingService failed. Will retry in the next run", omRequest, e); } } } + /** + * Checks if a given snapshot has been deleted and all the changes made to snapshot have been flushed to disk. + * @param snapInfo SnapshotInfo corresponding to the snapshot. + * @return true if the snapshot is still active or changes to snapshot have not been flushed to disk otherwise false. + * @throws IOException + */ @VisibleForTesting - boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) { + boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) throws IOException { SnapshotInfo.SnapshotStatus snapshotStatus = snapInfo.getSnapshotStatus(); - return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED; + return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo); } // TODO: Move this util class. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index e0f40dabd8a..7af6d085137 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -24,8 +24,11 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -33,6 +36,8 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.NoSuchElementException; import java.util.HashMap; import java.util.Map; @@ -86,6 +91,13 @@ public static SnapshotInfo getSnapshotInfo(final OzoneManager ozoneManager, return snapshotInfo; } + public static SnapshotInfo getSnapshotInfo(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + UUID snapshotId) throws IOException { + String tableKey = chainManager.getTableKey(snapshotId); + return SnapshotUtils.getSnapshotInfo(ozoneManager, tableKey); + } + public static void dropColumnFamilyHandle( final ManagedRocksDB rocksDB, final ColumnFamilyHandle columnFamilyHandle) { @@ -138,37 +150,25 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } + /** - * Get the next non deleted snapshot in the snapshot chain. + * Get the next snapshot in the snapshot chain. */ - public static SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, OzoneManager ozoneManager) + public static SnapshotInfo getNextSnapshot(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + SnapshotInfo snapInfo) throws IOException { - // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot // is removed in-memory but OMDoubleBuffer has not flushed yet. if (snapInfo == null) { throw new OMException("Snapshot Info is null. Cannot get the next snapshot", INVALID_SNAPSHOT_ERROR); } - try { - while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + if (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId())) { - - UUID nextPathSnapshot = - chainManager.nextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - - String tableKey = chainManager.getTableKey(nextPathSnapshot); - SnapshotInfo nextSnapshotInfo = getSnapshotInfo(ozoneManager, tableKey); - - if (nextSnapshotInfo.getSnapshotStatus().equals( - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { - return nextSnapshotInfo; - } - - snapInfo = nextSnapshotInfo; + UUID nextPathSnapshot = chainManager.nextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); + return getSnapshotInfo(ozoneManager, chainManager, nextPathSnapshot); } } catch (NoSuchElementException ex) { LOG.error("The snapshot {} is not longer in snapshot chain, It " + @@ -242,4 +242,44 @@ public static String getOzonePathKeyForFso(OMMetadataManager metadataManager, final long bucketId = metadataManager.getBucketId(volumeName, bucketName); return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX; } + + /** + * Returns merged repeatedKeyInfo entry with the existing deleted entry in the table. + * @param snapshotMoveKeyInfos keyInfos to be added. + * @param metadataManager metadataManager for a store. + * @return + * @throws IOException + */ + public static RepeatedOmKeyInfo createMergedRepeatedOmKeyInfoFromDeletedTableEntry( + OzoneManagerProtocolProtos.SnapshotMoveKeyInfos snapshotMoveKeyInfos, OMMetadataManager metadataManager) throws + IOException { + String dbKey = snapshotMoveKeyInfos.getKey(); + List keyInfoList = new ArrayList<>(); + for (OzoneManagerProtocolProtos.KeyInfo info : snapshotMoveKeyInfos.getKeyInfosList()) { + OmKeyInfo fromProtobuf = OmKeyInfo.getFromProtobuf(info); + keyInfoList.add(fromProtobuf); + } + // When older version of keys are moved to the next snapshot's deletedTable + // The newer version might also be in the next snapshot's deletedTable and + // it might overwrite the existing value which inturn could lead to orphan block in the system. + // Checking the keyInfoList with the last n versions of the omKeyInfo versions would ensure all versions are + // present in the list and would also avoid redundant additions to the list if the last n versions match, which + // can happen on om transaction replay on snapshotted rocksdb. + RepeatedOmKeyInfo result = metadataManager.getDeletedTable().get(dbKey); + if (result == null) { + result = new RepeatedOmKeyInfo(keyInfoList); + } else if (!isSameAsLatestOmKeyInfo(keyInfoList, result)) { + keyInfoList.forEach(result::addOmKeyInfo); + } + return result; + } + + private static boolean isSameAsLatestOmKeyInfo(List omKeyInfos, + RepeatedOmKeyInfo result) { + int size = result.getOmKeyInfoList().size(); + if (size >= omKeyInfos.size()) { + return omKeyInfos.equals(result.getOmKeyInfoList().subList(size - omKeyInfos.size(), size)); + } + return false; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index c807c04688d..eff23a18e6e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -30,9 +30,11 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import javax.xml.bind.DatatypeConverter; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; @@ -40,6 +42,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfigValidator; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.ozone.ClientVersion; @@ -109,6 +112,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.logging.log4j.util.Strings; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doCallRealMethod; @@ -749,17 +753,17 @@ public static OMRequest.Builder newCreateBucketRequest( .setClientId(UUID.randomUUID().toString()); } - public static List< HddsProtos.KeyValue> getMetadataList() { - List metadataList = new ArrayList<>(); - metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key1").setValue( + public static List< KeyValue> getMetadataList() { + List metadataList = new ArrayList<>(); + metadataList.add(KeyValue.newBuilder().setKey("key1").setValue( "value1").build()); - metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key2").setValue( + metadataList.add(KeyValue.newBuilder().setKey("key2").setValue( "value2").build()); return metadataList; } - public static HddsProtos.KeyValue fsoMetadata() { - return HddsProtos.KeyValue.newBuilder() + public static KeyValue fsoMetadata() { + return KeyValue.newBuilder() .setKey(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS) .setValue(Boolean.FALSE.toString()) .build(); @@ -1050,7 +1054,7 @@ public static OMRequest createCommitPartMPURequest(String volumeName, .setMultipartNumber(partNumber) .setMultipartUploadID(multipartUploadID) .addAllKeyLocations(new ArrayList<>()) - .addMetadata(HddsProtos.KeyValue.newBuilder() + .addMetadata(KeyValue.newBuilder() .setKey(OzoneConsts.ETAG) .setValue(DatatypeConverter.printHexBinary( new DigestInputStream( @@ -1321,6 +1325,69 @@ public static OMRequest createSnapshotRequest(String volumeName, .build(); } + public static OMRequest moveSnapshotTableKeyRequest(UUID snapshotId, + List>> deletedKeys, + List>> deletedDirs, + List> renameKeys) { + List deletedMoveKeys = new ArrayList<>(); + for (Pair> deletedKey : deletedKeys) { + OzoneManagerProtocolProtos.SnapshotMoveKeyInfos snapshotMoveKeyInfos = + OzoneManagerProtocolProtos.SnapshotMoveKeyInfos.newBuilder() + .setKey(deletedKey.getKey()) + .addAllKeyInfos( + deletedKey.getValue().stream() + .map(omKeyInfo -> omKeyInfo.getProtobuf(ClientVersion.CURRENT_VERSION)).collect(Collectors.toList())) + .build(); + deletedMoveKeys.add(snapshotMoveKeyInfos); + } + + List deletedDirMoveKeys = new ArrayList<>(); + for (Pair> deletedKey : deletedDirs) { + OzoneManagerProtocolProtos.SnapshotMoveKeyInfos snapshotMoveKeyInfos = + OzoneManagerProtocolProtos.SnapshotMoveKeyInfos.newBuilder() + .setKey(deletedKey.getKey()) + .addAllKeyInfos( + deletedKey.getValue().stream() + .map(omKeyInfo -> omKeyInfo.getProtobuf(ClientVersion.CURRENT_VERSION)) + .collect(Collectors.toList())) + .build(); + deletedDirMoveKeys.add(snapshotMoveKeyInfos); + } + + List renameKeyList = new ArrayList<>(); + for (Pair renameKey : renameKeys) { + KeyValue.Builder keyValue = KeyValue.newBuilder(); + keyValue.setKey(renameKey.getKey()); + if (!Strings.isBlank(renameKey.getValue())) { + keyValue.setValue(renameKey.getValue()); + } + renameKeyList.add(keyValue.build()); + } + + + OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest snapshotMoveTableKeysRequest = + OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest.newBuilder() + .setFromSnapshotID(HddsUtils.toProtobuf(snapshotId)) + .addAllDeletedKeys(deletedMoveKeys) + .addAllDeletedDirs(deletedDirMoveKeys) + .addAllRenamedKeys(renameKeyList) + .build(); + + OzoneManagerProtocolProtos.UserInfo userInfo = + OzoneManagerProtocolProtos.UserInfo.newBuilder() + .setUserName("user") + .setHostName("host") + .setRemoteAddress("remote-address") + .build(); + + return OMRequest.newBuilder() + .setSnapshotMoveTableKeysRequest(snapshotMoveTableKeysRequest) + .setCmdType(Type.SnapshotMoveTableKeys) + .setClientId(UUID.randomUUID().toString()) + .setUserInfo(userInfo) + .build(); + } + /** * Create OMRequest for Rename Snapshot. * diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 806c1b90f7f..af904382256 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -19,15 +19,8 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.TransactionInfo; -import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.audit.AuditMessage; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -38,18 +31,15 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponseWithFSO; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.io.File; import java.io.IOException; import java.util.UUID; @@ -65,69 +55,19 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.framework; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** * Tests OMSnapshotCreateRequest class, which handles CreateSnapshot request. */ -public class TestOMSnapshotCreateRequest { - @TempDir - private File anotherTempDir; - - private OzoneManager ozoneManager; - private OMMetrics omMetrics; - private OmMetadataManagerImpl omMetadataManager; - private BatchOperation batchOperation; - - private String volumeName; - private String bucketName; +public class TestOMSnapshotCreateRequest extends TestSnapshotRequestAndResponse { private String snapshotName1; private String snapshotName2; @BeforeEach public void setup() throws Exception { - ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, - anotherTempDir.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, - ozoneManager); - when(ozoneManager.getMetrics()).thenReturn(omMetrics); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - when(ozoneManager.isRatisEnabled()).thenReturn(true); - when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); - when(ozoneManager.isAdmin(any())).thenReturn(false); - when(ozoneManager.isOwner(any(), any())).thenReturn(false); - when(ozoneManager.getBucketOwner(any(), any(), - any(), any())).thenReturn("dummyBucketOwner"); - OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); - when(lvm.isAllowed(anyString())).thenReturn(true); - when(ozoneManager.getVersionManager()).thenReturn(lvm); - AuditLogger auditLogger = mock(AuditLogger.class); - when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); - doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); - batchOperation = omMetadataManager.getStore().initBatchOperation(); - - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); snapshotName1 = UUID.randomUUID().toString(); snapshotName2 = UUID.randomUUID().toString(); - OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, - omMetadataManager); - } - - @AfterEach - public void stop() { - omMetrics.unRegister(); - framework().clearInlineMocks(); - if (batchOperation != null) { - batchOperation.close(); - } } @ValueSource(strings = { @@ -140,9 +80,9 @@ public void stop() { }) @ParameterizedTest public void testPreExecute(String snapshotName) throws Exception { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); - OMRequest omRequest = createSnapshotRequest(volumeName, - bucketName, snapshotName); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName); doPreExecute(omRequest); } @@ -158,9 +98,9 @@ public void testPreExecute(String snapshotName) throws Exception { }) @ParameterizedTest public void testPreExecuteFailure(String snapshotName) { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); - OMRequest omRequest = createSnapshotRequest(volumeName, - bucketName, snapshotName); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); assertTrue(omException.getMessage() @@ -170,8 +110,8 @@ public void testPreExecuteFailure(String snapshotName) { @Test public void testPreExecuteBadOwner() { // Owner is not set for the request. - OMRequest omRequest = createSnapshotRequest(volumeName, - bucketName, snapshotName1); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName1); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); @@ -181,29 +121,29 @@ public void testPreExecuteBadOwner() { @Test public void testValidateAndUpdateCache() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); - OMRequest omRequest = createSnapshotRequest(volumeName, - bucketName, snapshotName1); + when(getOzoneManager().isAdmin(any())).thenReturn(true); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName1); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); - String key = getTableKey(volumeName, bucketName, snapshotName1); - String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + String key = getTableKey(getVolumeName(), getBucketName(), snapshotName1); + String bucketKey = getOmMetadataManager().getBucketKey(getVolumeName(), getBucketName()); // Add a 1000-byte key to the bucket OmKeyInfo key1 = addKey("key-testValidateAndUpdateCache", 12345L); addKeyToTable(key1); - OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( + OmBucketInfo omBucketInfo = getOmMetadataManager().getBucketTable().get( bucketKey); long bucketDataSize = key1.getDataSize(); long bucketUsedBytes = omBucketInfo.getUsedBytes(); assertEquals(key1.getReplicatedSize(), bucketUsedBytes); // Value in cache should be null as of now. - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // Run validateAndUpdateCache. OMClientResponse omClientResponse = - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1); assertNotNull(omClientResponse.getOMResponse()); @@ -227,21 +167,21 @@ public void testValidateAndUpdateCache() throws Exception { // Get value from cache SnapshotInfo snapshotInfoInCache = - omMetadataManager.getSnapshotInfoTable().get(key); + getOmMetadataManager().getSnapshotInfoTable().get(key); assertNotNull(snapshotInfoInCache); assertEquals(snapshotInfoFromProto, snapshotInfoInCache); assertEquals(snapshotInfoInCache.getLastTransactionInfo(), TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString()); - assertEquals(0, omMetrics.getNumSnapshotCreateFails()); - assertEquals(1, omMetrics.getNumSnapshotActive()); - assertEquals(1, omMetrics.getNumSnapshotCreates()); + assertEquals(0, getOmMetrics().getNumSnapshotCreateFails()); + assertEquals(1, getOmMetrics().getNumSnapshotActive()); + assertEquals(1, getOmMetrics().getNumSnapshotCreates()); } @Test public void testEntryRenamedKeyTable() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); Table snapshotRenamedTable = - omMetadataManager.getSnapshotRenamedTable(); + getOmMetadataManager().getSnapshotRenamedTable(); renameKey("key1", "key2", 0); renameDir("dir1", "dir2", 5); @@ -251,17 +191,17 @@ public void testEntryRenamedKeyTable() throws Exception { // Create snapshot createSnapshot(snapshotName1); - String snapKey = getTableKey(volumeName, - bucketName, snapshotName1); + String snapKey = getTableKey(getVolumeName(), + getBucketName(), snapshotName1); SnapshotInfo snapshotInfo = - omMetadataManager.getSnapshotInfoTable().get(snapKey); + getOmMetadataManager().getSnapshotInfoTable().get(snapKey); assertNotNull(snapshotInfo); renameKey("key3", "key4", 10); renameDir("dir3", "dir4", 15); // Rename table should have two entries as rename is within snapshot scope. - assertEquals(2, omMetadataManager + assertEquals(2, getOmMetadataManager() .countRowsInTable(snapshotRenamedTable)); // Create snapshot to clear snapshotRenamedTable @@ -271,33 +211,33 @@ public void testEntryRenamedKeyTable() throws Exception { @Test public void testEntryExists() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); - String key = getTableKey(volumeName, bucketName, snapshotName1); + String key = getTableKey(getVolumeName(), getBucketName(), snapshotName1); OMRequest omRequest = - createSnapshotRequest(volumeName, bucketName, snapshotName1); + createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1); - assertNotNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // Now try to create again to verify error - omRequest = createSnapshotRequest(volumeName, bucketName, snapshotName1); + omRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); omSnapshotCreateRequest = doPreExecute(omRequest); OMClientResponse omClientResponse = - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 2); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 2); OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse.getCreateSnapshotResponse()); assertEquals(OzoneManagerProtocolProtos.Status.FILE_ALREADY_EXISTS, omResponse.getStatus()); - assertEquals(1, omMetrics.getNumSnapshotCreateFails()); - assertEquals(1, omMetrics.getNumSnapshotActive()); - assertEquals(2, omMetrics.getNumSnapshotCreates()); + assertEquals(1, getOmMetrics().getNumSnapshotCreateFails()); + assertEquals(1, getOmMetrics().getNumSnapshotActive()); + assertEquals(2, getOmMetrics().getNumSnapshotCreates()); } private void renameKey(String fromKey, String toKey, long offset) @@ -316,15 +256,15 @@ private void renameKey(String fromKey, String toKey, long offset) new OMKeyRenameResponse(omResponse, fromKeyInfo.getKeyName(), toKeyInfo.getKeyName(), toKeyInfo); - omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + omKeyRenameResponse.addToDBBatch(getOmMetadataManager(), getBatchOperation()); + getOmMetadataManager().getStore().commitBatchOperation(getBatchOperation()); } private void renameDir(String fromKey, String toKey, long offset) throws Exception { String fromKeyParentName = UUID.randomUUID().toString(); - OmKeyInfo fromKeyParent = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, fromKeyParentName, RatisReplicationConfig.getInstance(THREE)) + OmKeyInfo fromKeyParent = OMRequestTestUtils.createOmKeyInfo(getVolumeName(), + getBucketName(), fromKeyParentName, RatisReplicationConfig.getInstance(THREE)) .setObjectID(100L) .build(); @@ -342,32 +282,32 @@ private void renameDir(String fromKey, String toKey, long offset) new OMKeyRenameResponseWithFSO(omResponse, getDBKeyName(fromKeyInfo), getDBKeyName(toKeyInfo), fromKeyParent, null, toKeyInfo, null, true, BucketLayout.FILE_SYSTEM_OPTIMIZED); - omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + omKeyRenameResponse.addToDBBatch(getOmMetadataManager(), getBatchOperation()); + getOmMetadataManager().getStore().commitBatchOperation(getBatchOperation()); } protected String getDBKeyName(OmKeyInfo keyInfo) throws IOException { - return omMetadataManager.getOzonePathKey( - omMetadataManager.getVolumeId(volumeName), - omMetadataManager.getBucketId(volumeName, bucketName), + return getOmMetadataManager().getOzonePathKey( + getOmMetadataManager().getVolumeId(getVolumeName()), + getOmMetadataManager().getBucketId(getVolumeName(), getBucketName()), keyInfo.getParentObjectID(), keyInfo.getKeyName()); } private void createSnapshot(String snapName) throws Exception { OMRequest omRequest = createSnapshotRequest( - volumeName, bucketName, snapName); + getVolumeName(), getBucketName(), snapName); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); //create entry OMClientResponse omClientResponse = - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); - omClientResponse.checkAndUpdateDB(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1); + omClientResponse.checkAndUpdateDB(getOmMetadataManager(), getBatchOperation()); + getOmMetadataManager().getStore().commitBatchOperation(getBatchOperation()); } private OMSnapshotCreateRequest doPreExecute( OMRequest originalRequest) throws Exception { - return doPreExecute(originalRequest, ozoneManager); + return doPreExecute(originalRequest, getOzoneManager()); } /** @@ -384,15 +324,15 @@ public static OMSnapshotCreateRequest doPreExecute( } private OmKeyInfo addKey(String keyName, long objectId) { - return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + return OMRequestTestUtils.createOmKeyInfo(getVolumeName(), getBucketName(), keyName, RatisReplicationConfig.getInstance(THREE)).setObjectID(objectId) .build(); } protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception { OMRequestTestUtils.addKeyToTable(false, true, keyInfo, 0, 0L, - omMetadataManager); - return omMetadataManager.getOzoneKey(keyInfo.getVolumeName(), + getOmMetadataManager()); + return getOmMetadataManager().getOzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 5a8bb5d7c0d..4c5dc2e77f0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -19,33 +19,21 @@ package org.apache.hadoop.ozone.om.request.snapshot; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.audit.AuditMessage; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.io.File; import java.util.UUID; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; @@ -61,10 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.framework; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** @@ -72,60 +56,15 @@ * Mostly mirrors TestOMSnapshotCreateRequest. * testEntryNotExist() and testEntryExists() are unique. */ -public class TestOMSnapshotDeleteRequest { - @TempDir - private File folder; +public class TestOMSnapshotDeleteRequest extends TestSnapshotRequestAndResponse { - private OzoneManager ozoneManager; - private OMMetrics omMetrics; - private OmMetadataManagerImpl omMetadataManager; - - private String volumeName; - private String bucketName; private String snapshotName; @BeforeEach public void setup() throws Exception { - ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, - folder.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, - ozoneManager); - when(ozoneManager.getMetrics()).thenReturn(omMetrics); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - when(ozoneManager.isRatisEnabled()).thenReturn(true); - when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); - when(ozoneManager.isAdmin(any())).thenReturn(false); - when(ozoneManager.isOwner(any(), any())).thenReturn(false); - when(ozoneManager.getBucketOwner(any(), any(), - any(), any())).thenReturn("dummyBucketOwner"); - OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); - when(lvm.isAllowed(anyString())).thenReturn(true); - when(ozoneManager.getVersionManager()).thenReturn(lvm); - AuditLogger auditLogger = mock(AuditLogger.class); - when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); - doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); - - OmSnapshotManager omSnapshotManager = mock(OmSnapshotManager.class); - when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); - - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); snapshotName = UUID.randomUUID().toString(); - OMRequestTestUtils.addVolumeAndBucketToDB( - volumeName, bucketName, omMetadataManager); - } - @AfterEach - public void stop() { - omMetrics.unRegister(); - framework().clearInlineMocks(); - } - - @ValueSource(strings = { // '-' is allowed. "9cdf0e8a-6946-41ad-a2d1-9eb724fab126", @@ -136,9 +75,9 @@ public void stop() { }) @ParameterizedTest public void testPreExecute(String deleteSnapshotName) throws Exception { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); - OMRequest omRequest = deleteSnapshotRequest(volumeName, - bucketName, deleteSnapshotName); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + OMRequest omRequest = deleteSnapshotRequest(getVolumeName(), + getBucketName(), deleteSnapshotName); doPreExecute(omRequest); } @@ -154,9 +93,9 @@ public void testPreExecute(String deleteSnapshotName) throws Exception { }) @ParameterizedTest public void testPreExecuteFailure(String deleteSnapshotName) { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); - OMRequest omRequest = deleteSnapshotRequest(volumeName, - bucketName, deleteSnapshotName); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + OMRequest omRequest = deleteSnapshotRequest(getVolumeName(), + getBucketName(), deleteSnapshotName); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); assertTrue(omException.getMessage() @@ -166,8 +105,8 @@ public void testPreExecuteFailure(String deleteSnapshotName) { @Test public void testPreExecuteBadOwner() { // Owner is not set for the request. - OMRequest omRequest = deleteSnapshotRequest(volumeName, - bucketName, snapshotName); + OMRequest omRequest = deleteSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); @@ -177,27 +116,27 @@ public void testPreExecuteBadOwner() { @Test public void testValidateAndUpdateCache() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); OMRequest omRequest = - deleteSnapshotRequest(volumeName, bucketName, snapshotName); + deleteSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); OMSnapshotDeleteRequest omSnapshotDeleteRequest = doPreExecute(omRequest); - String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); + String key = SnapshotInfo.getTableKey(getVolumeName(), getBucketName(), snapshotName); // As we have not still called validateAndUpdateCache, get() should // return null. - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // add key to cache - SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(volumeName, bucketName, + SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(getVolumeName(), getBucketName(), snapshotName, null, Time.now()); assertEquals(SNAPSHOT_ACTIVE, snapshotInfo.getSnapshotStatus()); - omMetadataManager.getSnapshotInfoTable().addCacheEntry( + getOmMetadataManager().getSnapshotInfoTable().addCacheEntry( new CacheKey<>(key), CacheValue.get(1L, snapshotInfo)); // Trigger validateAndUpdateCache OMClientResponse omClientResponse = - omSnapshotDeleteRequest.validateAndUpdateCache(ozoneManager, 2L); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), 2L); OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse); @@ -207,14 +146,14 @@ public void testValidateAndUpdateCache() throws Exception { assertEquals(OK, omResponse.getStatus()); // check cache - snapshotInfo = omMetadataManager.getSnapshotInfoTable().get(key); + snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(key); assertNotNull(snapshotInfo); assertEquals(SNAPSHOT_DELETED, snapshotInfo.getSnapshotStatus()); - assertEquals(0, omMetrics.getNumSnapshotCreates()); + assertEquals(0, getOmMetrics().getNumSnapshotCreates()); // Expected -1 because no snapshot was created before. - assertEquals(-1, omMetrics.getNumSnapshotActive()); - assertEquals(1, omMetrics.getNumSnapshotDeleted()); - assertEquals(0, omMetrics.getNumSnapshotDeleteFails()); + assertEquals(-1, getOmMetrics().getNumSnapshotActive()); + assertEquals(1, getOmMetrics().getNumSnapshotDeleted()); + assertEquals(0, getOmMetrics().getNumSnapshotDeleteFails()); } /** @@ -222,25 +161,25 @@ public void testValidateAndUpdateCache() throws Exception { */ @Test public void testEntryNotExist() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); OMRequest omRequest = deleteSnapshotRequest( - volumeName, bucketName, snapshotName); + getVolumeName(), getBucketName(), snapshotName); OMSnapshotDeleteRequest omSnapshotDeleteRequest = doPreExecute(omRequest); - String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); + String key = SnapshotInfo.getTableKey(getVolumeName(), getBucketName(), snapshotName); // Entry does not exist - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // Trigger delete snapshot validateAndUpdateCache OMClientResponse omClientResponse = - omSnapshotDeleteRequest.validateAndUpdateCache(ozoneManager, 1L); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), 1L); OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse.getDeleteSnapshotResponse()); assertEquals(Status.FILE_NOT_FOUND, omResponse.getStatus()); - assertEquals(0, omMetrics.getNumSnapshotActive()); - assertEquals(0, omMetrics.getNumSnapshotDeleted()); - assertEquals(1, omMetrics.getNumSnapshotDeleteFails()); + assertEquals(0, getOmMetrics().getNumSnapshotActive()); + assertEquals(0, getOmMetrics().getNumSnapshotDeleted()); + assertEquals(1, getOmMetrics().getNumSnapshotDeleteFails()); } /** @@ -249,50 +188,50 @@ public void testEntryNotExist() throws Exception { */ @Test public void testEntryExist() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); - String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); + when(getOzoneManager().isAdmin(any())).thenReturn(true); + String key = SnapshotInfo.getTableKey(getVolumeName(), getBucketName(), snapshotName); OMRequest omRequest1 = - createSnapshotRequest(volumeName, bucketName, snapshotName); + createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); OMSnapshotCreateRequest omSnapshotCreateRequest = - TestOMSnapshotCreateRequest.doPreExecute(omRequest1, ozoneManager); + TestOMSnapshotCreateRequest.doPreExecute(omRequest1, getOzoneManager()); - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // Create snapshot entry - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1L); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1L); SnapshotInfo snapshotInfo = - omMetadataManager.getSnapshotInfoTable().get(key); + getOmMetadataManager().getSnapshotInfoTable().get(key); assertNotNull(snapshotInfo); assertEquals(SNAPSHOT_ACTIVE, snapshotInfo.getSnapshotStatus()); - assertEquals(1, omMetrics.getNumSnapshotActive()); + assertEquals(1, getOmMetrics().getNumSnapshotActive()); OMRequest omRequest2 = - deleteSnapshotRequest(volumeName, bucketName, snapshotName); + deleteSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); OMSnapshotDeleteRequest omSnapshotDeleteRequest = doPreExecute(omRequest2); // Delete snapshot entry OMClientResponse omClientResponse = - omSnapshotDeleteRequest.validateAndUpdateCache(ozoneManager, 2L); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), 2L); // Response should be successful OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse); assertNotNull(omResponse.getDeleteSnapshotResponse()); assertEquals(OK, omResponse.getStatus()); - snapshotInfo = omMetadataManager.getSnapshotInfoTable().get(key); + snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(key); // The snapshot entry should still exist in the table, // but marked as DELETED. assertNotNull(snapshotInfo); assertEquals(SNAPSHOT_DELETED, snapshotInfo.getSnapshotStatus()); assertThat(snapshotInfo.getDeletionTime()).isGreaterThan(0L); - assertEquals(0, omMetrics.getNumSnapshotActive()); + assertEquals(0, getOmMetrics().getNumSnapshotActive()); // Now delete snapshot entry again, expect error. - omRequest2 = deleteSnapshotRequest(volumeName, bucketName, snapshotName); + omRequest2 = deleteSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); omSnapshotDeleteRequest = doPreExecute(omRequest2); omClientResponse = - omSnapshotDeleteRequest.validateAndUpdateCache(ozoneManager, 3L); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), 3L); omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse); @@ -300,11 +239,11 @@ public void testEntryExist() throws Exception { assertEquals(Status.FILE_NOT_FOUND, omResponse.getStatus()); // Snapshot entry should still be there. - snapshotInfo = omMetadataManager.getSnapshotInfoTable().get(key); + snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(key); assertNotNull(snapshotInfo); assertEquals(SNAPSHOT_DELETED, snapshotInfo.getSnapshotStatus()); - assertEquals(0, omMetrics.getNumSnapshotActive()); - assertEquals(1, omMetrics.getNumSnapshotDeleteFails()); + assertEquals(0, getOmMetrics().getNumSnapshotActive()); + assertEquals(1, getOmMetrics().getNumSnapshotDeleteFails()); } private OMSnapshotDeleteRequest doPreExecute( @@ -313,7 +252,7 @@ private OMSnapshotDeleteRequest doPreExecute( new OMSnapshotDeleteRequest(originalRequest); OMRequest modifiedRequest = - omSnapshotDeleteRequest.preExecute(ozoneManager); + omSnapshotDeleteRequest.preExecute(getOzoneManager()); return new OMSnapshotDeleteRequest(modifiedRequest); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotMoveTableKeysRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotMoveTableKeysRequest.java new file mode 100644 index 00000000000..247f322dfcf --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotMoveTableKeysRequest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.deleteSnapshotRequest; +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.moveSnapshotTableKeyRequest; + +/** + * Class to test OmSnapshotMoveTableKeyRequest. + */ +public class TestOMSnapshotMoveTableKeysRequest extends TestSnapshotRequestAndResponse { + + private String snapshotName1; + private String snapshotName2; + private SnapshotInfo snapshotInfo1; + private SnapshotInfo snapshotInfo2; + + @BeforeEach + public void setup() throws Exception { + snapshotName1 = UUID.randomUUID().toString(); + snapshotName2 = UUID.randomUUID().toString(); + } + + public TestOMSnapshotMoveTableKeysRequest() { + super(true); + } + + private void createSnapshots(boolean createSecondSnapshot) throws Exception { + createSnapshotCheckpoint(getVolumeName(), getBucketName(), snapshotName1); + snapshotInfo1 = SnapshotUtils.getSnapshotInfo(getOzoneManager(), getVolumeName(), getBucketName(), snapshotName1); + if (createSecondSnapshot) { + createSnapshotCheckpoint(getVolumeName(), getBucketName(), snapshotName2); + snapshotInfo2 = SnapshotUtils.getSnapshotInfo(getOzoneManager(), getVolumeName(), getBucketName(), snapshotName2); + } + } + + private SnapshotInfo deleteSnapshot(SnapshotInfo snapshotInfo, long transactionIndex) throws Exception { + OzoneManagerProtocolProtos.OMRequest omRequest = deleteSnapshotRequest(snapshotInfo.getVolumeName(), + snapshotInfo.getBucketName(), snapshotInfo.getName()); + OMSnapshotDeleteRequest omSnapshotDeleteRequest = new OMSnapshotDeleteRequest(omRequest); + omSnapshotDeleteRequest.preExecute(getOzoneManager()); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), transactionIndex); + return SnapshotUtils.getSnapshotInfo(getOzoneManager(), snapshotInfo.getTableKey()); + } + + @Test + public void testValidateAndUpdateCacheWithNextSnapshotInactive() throws Exception { + createSnapshots(true); + snapshotInfo2 = deleteSnapshot(snapshotInfo2, 0); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest( + omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + OMClientResponse omClientResponse = omSnapshotMoveTableKeysRequest.validateAndUpdateCache(getOzoneManager(), 1); + Assertions.assertFalse(omClientResponse.getOMResponse().getSuccess()); + Assertions.assertEquals(OzoneManagerProtocolProtos.Status.INVALID_SNAPSHOT_ERROR, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testPreExecuteWithInvalidDeletedKeyPrefix() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedKeys = + Stream.of(getDeletedKeys(getVolumeName(), getBucketName(), 0, 10, 10, 0), + getDeletedKeys(invalidVolumeName, invalidBucketName, 0, 10, 10, 0)) + .flatMap(List::stream).collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + deletedKeys, Collections.emptyList(), Collections.emptyList()); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_KEY_NAME, omException.getResult()); + } + + @Test + public void testPreExecuteWithInvalidDeletedDirPrefix() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedDirs = + Stream.of(getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1), + getDeletedDirKeys(invalidVolumeName, invalidBucketName, 0, 10, 1)) + .flatMap(List::stream).collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + Collections.emptyList(), deletedDirs, Collections.emptyList()); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_KEY_NAME, omException.getResult()); + } + + @Test + public void testPreExecuteWithInvalidNumberKeys() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedDirs = + Stream.of(getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1), + getDeletedDirKeys(invalidVolumeName, invalidBucketName, 0, 10, 10)) + .flatMap(List::stream).collect(Collectors.toList()); + List>> deletedKeys = + Stream.of(getDeletedKeys(getVolumeName(), getBucketName(), 0, 10, 10, 0), + getDeletedKeys(invalidVolumeName, invalidBucketName, 0, 10, 0, 0)) + .flatMap(List::stream).collect(Collectors.toList()); + List> renameKeys = getRenameKeys(getVolumeName(), getBucketName(), 0, 10, snapshotName1); + renameKeys.add(Pair.of(getOmMetadataManager().getRenameKey(getVolumeName(), getBucketName(), 11), null)); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + deletedKeys, deletedDirs, renameKeys); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + omRequest = omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager()); + for (OzoneManagerProtocolProtos.SnapshotMoveKeyInfos deletedDir : + omRequest.getSnapshotMoveTableKeysRequest().getDeletedDirsList()) { + Assertions.assertEquals(1, deletedDir.getKeyInfosList().size()); + } + + for (OzoneManagerProtocolProtos.SnapshotMoveKeyInfos deletedKey : + omRequest.getSnapshotMoveTableKeysRequest().getDeletedKeysList()) { + Assertions.assertNotEquals(0, deletedKey.getKeyInfosList().size()); + } + + for (HddsProtos.KeyValue renameKey : omRequest.getSnapshotMoveTableKeysRequest().getRenamedKeysList()) { + Assertions.assertTrue(renameKey.hasKey() && renameKey.hasValue()); + } + + } + + @Test + public void testPreExecuteWithInvalidRenamePrefix() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List> renameKeys = + Stream.of(getRenameKeys(getVolumeName(), getBucketName(), 0, 10, snapshotName1), + getRenameKeys(invalidVolumeName, invalidBucketName, 0, 10, snapshotName2)).flatMap(List::stream) + .collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + Collections.emptyList(), Collections.emptyList(), renameKeys); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_KEY_NAME, omException.getResult()); + } + + @Test + public void testValidateAndUpdateCache() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedKeys = getDeletedKeys(getVolumeName(), getBucketName(), 0, 10, 10, 0); + List>> deletedDirs = getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1); + List> renameKeys = getRenameKeys(getVolumeName(), getBucketName(), 0, 10, snapshotName1); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + deletedKeys, deletedDirs, renameKeys); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + // perform preExecute. + omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest( + omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + OMClientResponse omClientResponse = omSnapshotMoveTableKeysRequest.validateAndUpdateCache(getOzoneManager(), 1); + Assertions.assertTrue(omClientResponse.getOMResponse().getSuccess()); + Assertions.assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testPreExecuteWithInvalidDuplicateDeletedKey() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedKeys = + Stream.of(getDeletedKeys(getVolumeName(), getBucketName(), 0, 10, 10, 0), + getDeletedKeys(getVolumeName(), getBucketName(), 0, 10, 10, 0)).flatMap(List::stream) + .collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + deletedKeys, Collections.emptyList(), Collections.emptyList()); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_REQUEST, omException.getResult()); + } + + @Test + public void testPreExecuteWithInvalidDuplicateDeletedDir() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List>> deletedDirs = + Stream.of(getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1), + getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1)).flatMap(List::stream) + .collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + Collections.emptyList(), deletedDirs, Collections.emptyList()); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_REQUEST, omException.getResult()); + } + + @Test + public void testPreExecuteWithInvalidDuplicateRenameKey() throws Exception { + createSnapshots(true); + String invalidVolumeName = UUID.randomUUID().toString(); + String invalidBucketName = UUID.randomUUID().toString(); + addVolumeAndBucketToDB(invalidVolumeName, invalidBucketName, getOmMetadataManager()); + List> renameKeys = + Stream.of(getRenameKeys(getVolumeName(), getBucketName(), 0, 10, snapshotName1), + getRenameKeys(getVolumeName(), getBucketName(), 0, 10, snapshotName1)) + .flatMap(List::stream).collect(Collectors.toList()); + OzoneManagerProtocolProtos.OMRequest omRequest = moveSnapshotTableKeyRequest(snapshotInfo1.getSnapshotId(), + Collections.emptyList(), Collections.emptyList(), renameKeys); + OMSnapshotMoveTableKeysRequest omSnapshotMoveTableKeysRequest = new OMSnapshotMoveTableKeysRequest(omRequest); + OMException omException = Assertions.assertThrows(OMException.class, + () -> omSnapshotMoveTableKeysRequest.preExecute(getOzoneManager())); + Assertions.assertEquals(INVALID_REQUEST, omException.getResult()); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index d0a5559a87b..1c44decdfda 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -20,43 +20,28 @@ package org.apache.hadoop.ozone.om.request.snapshot; import com.google.protobuf.ByteString; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; -import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.om.IOmMetadataReader; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -71,10 +56,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -82,49 +65,16 @@ /** * Tests OMSnapshotPurgeRequest class. */ -public class TestOMSnapshotPurgeRequestAndResponse { - private List checkpointPaths = new ArrayList<>(); - - private OzoneManager ozoneManager; - private OMMetrics omMetrics; - private OMMetadataManager omMetadataManager; - private OmSnapshotManager omSnapshotManager; - private AuditLogger auditLogger; - - private String volumeName; - private String bucketName; +public class TestOMSnapshotPurgeRequestAndResponse extends TestSnapshotRequestAndResponse { + private final List checkpointPaths = new ArrayList<>(); private String keyName; + public TestOMSnapshotPurgeRequestAndResponse() { + super(true); + } + @BeforeEach - void setup(@TempDir File testDir) throws Exception { - ozoneManager = mock(OzoneManager.class); - OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); - when(lvm.isAllowed(anyString())).thenReturn(true); - when(ozoneManager.getVersionManager()).thenReturn(lvm); - when(ozoneManager.isRatisEnabled()).thenReturn(true); - auditLogger = mock(AuditLogger.class); - when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); - omMetrics = OMMetrics.create(); - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, - testDir.getAbsolutePath()); - ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - testDir.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, - ozoneManager); - when(ozoneManager.getMetrics()).thenReturn(omMetrics); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); - when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); - - ReferenceCounted rcOmMetadataReader = - mock(ReferenceCounted.class); - when(ozoneManager.getOmMetadataReader()).thenReturn(rcOmMetadataReader); - omSnapshotManager = new OmSnapshotManager(ozoneManager); - when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); + public void setup() throws Exception { keyName = UUID.randomUUID().toString(); } @@ -135,17 +85,14 @@ private List createSnapshots(int numSnapshotKeys) throws Exception { Random random = new Random(); - // Add volume, bucket and key entries to OM DB. - OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, - omMetadataManager); // Create Snapshot and CheckpointDir List purgeSnapshots = new ArrayList<>(numSnapshotKeys); for (int i = 1; i <= numSnapshotKeys; i++) { String snapshotName = keyName + "-" + random.nextLong(); createSnapshotCheckpoint(snapshotName); - purgeSnapshots.add(SnapshotInfo.getTableKey(volumeName, - bucketName, snapshotName)); + purgeSnapshots.add(SnapshotInfo.getTableKey(getVolumeName(), + getBucketName(), snapshotName)); } return purgeSnapshots; @@ -175,39 +122,7 @@ private OMRequest createPurgeKeysRequest(List purgeSnapshotKeys) { * Create snapshot and checkpoint directory. */ private void createSnapshotCheckpoint(String snapshotName) throws Exception { - createSnapshotCheckpoint(volumeName, bucketName, snapshotName); - } - - private void createSnapshotCheckpoint(String volume, - String bucket, - String snapshotName) throws Exception { - OMRequest omRequest = OMRequestTestUtils - .createSnapshotRequest(volume, bucket, snapshotName); - // Pre-Execute OMSnapshotCreateRequest. - OMSnapshotCreateRequest omSnapshotCreateRequest = - TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager); - - // validateAndUpdateCache OMSnapshotCreateResponse. - OMSnapshotCreateResponse omClientResponse = (OMSnapshotCreateResponse) - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); - // Add to batch and commit to DB. - try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - omClientResponse.addToDBBatch(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); - } - - String key = SnapshotInfo.getTableKey(volume, bucket, snapshotName); - SnapshotInfo snapshotInfo = - omMetadataManager.getSnapshotInfoTable().get(key); - assertNotNull(snapshotInfo); - - RDBStore store = (RDBStore) omMetadataManager.getStore(); - String checkpointPrefix = store.getDbLocation().getName(); - Path snapshotDirPath = Paths.get(store.getSnapshotsParentDir(), - checkpointPrefix + snapshotInfo.getCheckpointDir()); - // Check the DB is still there - assertTrue(Files.exists(snapshotDirPath)); - checkpointPaths.add(snapshotDirPath); + checkpointPaths.add(createSnapshotCheckpoint(getVolumeName(), getBucketName(), snapshotName)); } private OMSnapshotPurgeRequest preExecute(OMRequest originalOmRequest) @@ -215,7 +130,7 @@ private OMSnapshotPurgeRequest preExecute(OMRequest originalOmRequest) OMSnapshotPurgeRequest omSnapshotPurgeRequest = new OMSnapshotPurgeRequest(originalOmRequest); OMRequest modifiedOmRequest = omSnapshotPurgeRequest - .preExecute(ozoneManager); + .preExecute(getOzoneManager()); return new OMSnapshotPurgeRequest(modifiedOmRequest); } @@ -227,48 +142,48 @@ private void purgeSnapshots(OMRequest snapshotPurgeRequest) // validateAndUpdateCache for OMSnapshotPurgeRequest. OMSnapshotPurgeResponse omSnapshotPurgeResponse = (OMSnapshotPurgeResponse) - omSnapshotPurgeRequest.validateAndUpdateCache(ozoneManager, 200L); + omSnapshotPurgeRequest.validateAndUpdateCache(getOzoneManager(), 200L); // Commit to DB. - try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - omSnapshotPurgeResponse.checkAndUpdateDB(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + try (BatchOperation batchOperation = getOmMetadataManager().getStore().initBatchOperation()) { + omSnapshotPurgeResponse.checkAndUpdateDB(getOmMetadataManager(), batchOperation); + getOmMetadataManager().getStore().commitBatchOperation(batchOperation); } } @Test public void testValidateAndUpdateCache() throws Exception { - long initialSnapshotPurgeCount = omMetrics.getNumSnapshotPurges(); - long initialSnapshotPurgeFailCount = omMetrics.getNumSnapshotPurgeFails(); + long initialSnapshotPurgeCount = getOmMetrics().getNumSnapshotPurges(); + long initialSnapshotPurgeFailCount = getOmMetrics().getNumSnapshotPurgeFails(); List snapshotDbKeysToPurge = createSnapshots(10); - assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); + assertFalse(getOmMetadataManager().getSnapshotInfoTable().isEmpty()); OMRequest snapshotPurgeRequest = createPurgeKeysRequest( snapshotDbKeysToPurge); OMSnapshotPurgeRequest omSnapshotPurgeRequest = preExecute(snapshotPurgeRequest); OMSnapshotPurgeResponse omSnapshotPurgeResponse = (OMSnapshotPurgeResponse) - omSnapshotPurgeRequest.validateAndUpdateCache(ozoneManager, 200L); + omSnapshotPurgeRequest.validateAndUpdateCache(getOzoneManager(), 200L); for (String snapshotTableKey: snapshotDbKeysToPurge) { - assertNull(omMetadataManager.getSnapshotInfoTable().get(snapshotTableKey)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(snapshotTableKey)); } - try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - omSnapshotPurgeResponse.checkAndUpdateDB(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + try (BatchOperation batchOperation = getOmMetadataManager().getStore().initBatchOperation()) { + omSnapshotPurgeResponse.checkAndUpdateDB(getOmMetadataManager(), batchOperation); + getOmMetadataManager().getStore().commitBatchOperation(batchOperation); } // Check if the entries are deleted. - assertTrue(omMetadataManager.getSnapshotInfoTable().isEmpty()); + assertTrue(getOmMetadataManager().getSnapshotInfoTable().isEmpty()); // Check if all the checkpoints are cleared. for (Path checkpoint : checkpointPaths) { assertFalse(Files.exists(checkpoint)); } - assertEquals(initialSnapshotPurgeCount + 1, omMetrics.getNumSnapshotPurges()); - assertEquals(initialSnapshotPurgeFailCount, omMetrics.getNumSnapshotPurgeFails()); + assertEquals(initialSnapshotPurgeCount + 1, getOmMetrics().getNumSnapshotPurges()); + assertEquals(initialSnapshotPurgeFailCount, getOmMetrics().getNumSnapshotPurgeFails()); } /** @@ -276,8 +191,8 @@ public void testValidateAndUpdateCache() throws Exception { */ @Test public void testValidateAndUpdateCacheFailure() throws Exception { - long initialSnapshotPurgeCount = omMetrics.getNumSnapshotPurges(); - long initialSnapshotPurgeFailCount = omMetrics.getNumSnapshotPurgeFails(); + long initialSnapshotPurgeCount = getOmMetrics().getNumSnapshotPurges(); + long initialSnapshotPurgeFailCount = getOmMetrics().getNumSnapshotPurgeFails(); List snapshotDbKeysToPurge = createSnapshots(10); @@ -286,17 +201,17 @@ public void testValidateAndUpdateCacheFailure() throws Exception { when(mockedSnapshotInfoTable.get(anyString())).thenThrow(new IOException("Injected fault error.")); when(mockedMetadataManager.getSnapshotInfoTable()).thenReturn(mockedSnapshotInfoTable); - when(ozoneManager.getMetadataManager()).thenReturn(mockedMetadataManager); + when(getOzoneManager().getMetadataManager()).thenReturn(mockedMetadataManager); OMRequest snapshotPurgeRequest = createPurgeKeysRequest(snapshotDbKeysToPurge); OMSnapshotPurgeRequest omSnapshotPurgeRequest = preExecute(snapshotPurgeRequest); OMSnapshotPurgeResponse omSnapshotPurgeResponse = (OMSnapshotPurgeResponse) - omSnapshotPurgeRequest.validateAndUpdateCache(ozoneManager, 200L); + omSnapshotPurgeRequest.validateAndUpdateCache(getOzoneManager(), 200L); assertEquals(INTERNAL_ERROR, omSnapshotPurgeResponse.getOMResponse().getStatus()); - assertEquals(initialSnapshotPurgeCount, omMetrics.getNumSnapshotPurges()); - assertEquals(initialSnapshotPurgeFailCount + 1, omMetrics.getNumSnapshotPurgeFails()); + assertEquals(initialSnapshotPurgeCount, getOmMetrics().getNumSnapshotPurges()); + assertEquals(initialSnapshotPurgeFailCount + 1, getOmMetrics().getNumSnapshotPurgeFails()); } // TODO: clean up: Do we this test after @@ -309,7 +224,7 @@ public void testSnapshotChainCleanup(int index) throws Exception { // Before purge, check snapshot chain OmMetadataManagerImpl metadataManager = - (OmMetadataManagerImpl) omMetadataManager; + (OmMetadataManagerImpl) getOmMetadataManager(); SnapshotChainManager chainManager = metadataManager .getSnapshotChainManager(); SnapshotInfo snapInfo = metadataManager.getSnapshotInfoTable() @@ -343,8 +258,8 @@ public void testSnapshotChainCleanup(int index) throws Exception { snapInfo.getSnapshotId()); } - long rowsInTableBeforePurge = omMetadataManager - .countRowsInTable(omMetadataManager.getSnapshotInfoTable()); + long rowsInTableBeforePurge = getOmMetadataManager() + .countRowsInTable(getOmMetadataManager().getSnapshotInfoTable()); // Purge Snapshot of the given index. List toPurgeList = Collections.singletonList(snapShotToPurge); OMRequest snapshotPurgeRequest = createPurgeKeysRequest( @@ -367,8 +282,8 @@ public void testSnapshotChainCleanup(int index) throws Exception { .getGlobalPreviousSnapshotId(), prevGlobalSnapId); } - assertNotEquals(rowsInTableBeforePurge, omMetadataManager - .countRowsInTable(omMetadataManager.getSnapshotInfoTable())); + assertNotEquals(rowsInTableBeforePurge, getOmMetadataManager() + .countRowsInTable(getOmMetadataManager().getSnapshotInfoTable())); } private static Stream snapshotPurgeCases() { @@ -422,14 +337,14 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( int toIndex, boolean createInBucketOrder) throws Exception { SnapshotChainManager chainManager = - ((OmMetadataManagerImpl) omMetadataManager).getSnapshotChainManager(); + ((OmMetadataManagerImpl) getOmMetadataManager()).getSnapshotChainManager(); int totalKeys = numberOfBuckets * numberOfKeysPerBucket; List buckets = new ArrayList<>(); for (int i = 0; i < numberOfBuckets; i++) { String bucketNameLocal = "bucket-" + UUID.randomUUID(); - OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketNameLocal, - omMetadataManager); + OMRequestTestUtils.addVolumeAndBucketToDB(getVolumeName(), bucketNameLocal, + getOmMetadataManager()); buckets.add(bucketNameLocal); } @@ -440,17 +355,17 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( int bucketIndex = createInBucketOrder ? i : j; String bucket = buckets.get(bucketIndex % numberOfBuckets); String snapshotName = UUID.randomUUID().toString(); - createSnapshotCheckpoint(volumeName, bucket, snapshotName); + createSnapshotCheckpoint(getVolumeName(), bucket, snapshotName); String snapshotTableKey = - SnapshotInfo.getTableKey(volumeName, bucket, snapshotName); + SnapshotInfo.getTableKey(getVolumeName(), bucket, snapshotName); SnapshotInfo snapshotInfo = - omMetadataManager.getSnapshotInfoTable().get(snapshotTableKey); + getOmMetadataManager().getSnapshotInfoTable().get(snapshotTableKey); snapshotInfoList.add(snapshotInfo); } } - long numberOfSnapshotBeforePurge = omMetadataManager - .countRowsInTable(omMetadataManager.getSnapshotInfoTable()); + long numberOfSnapshotBeforePurge = getOmMetadataManager() + .countRowsInTable(getOmMetadataManager().getSnapshotInfoTable()); assertEquals(totalKeys, numberOfSnapshotBeforePurge); assertEquals(totalKeys, chainManager.getGlobalSnapshotChain().size()); Map expectedTransactionInfos = new HashMap<>(); @@ -476,7 +391,7 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( expectedTransactionInfos.put(chainManager.nextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId), expectedLastTransactionVal); } - String purgeSnapshotKey = SnapshotInfo.getTableKey(volumeName, + String purgeSnapshotKey = SnapshotInfo.getTableKey(getVolumeName(), purgeSnapshotInfo.getBucketName(), purgeSnapshotInfo.getName()); purgeSnapshotKeys.add(purgeSnapshotKey); @@ -489,17 +404,17 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( for (int i = 0; i < totalKeys; i++) { if (i < fromIndex || i > toIndex) { SnapshotInfo info = snapshotInfoList.get(i); - String snapshotKey = SnapshotInfo.getTableKey(volumeName, + String snapshotKey = SnapshotInfo.getTableKey(getVolumeName(), info.getBucketName(), info.getName()); snapshotInfoListAfterPurge.add( - omMetadataManager.getSnapshotInfoTable().get(snapshotKey)); + getOmMetadataManager().getSnapshotInfoTable().get(snapshotKey)); } } long expectNumberOfSnapshotAfterPurge = totalKeys - (toIndex - fromIndex + 1); - long actualNumberOfSnapshotAfterPurge = omMetadataManager - .countRowsInTable(omMetadataManager.getSnapshotInfoTable()); + long actualNumberOfSnapshotAfterPurge = getOmMetadataManager() + .countRowsInTable(getOmMetadataManager().getSnapshotInfoTable()); assertEquals(expectNumberOfSnapshotAfterPurge, actualNumberOfSnapshotAfterPurge); assertEquals(expectNumberOfSnapshotAfterPurge, chainManager @@ -516,7 +431,7 @@ private void validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain( assertEquals(snapshotInfo.getLastTransactionInfo(), expectedTransactionInfos.get(snapshotInfo.getSnapshotId())); } OmMetadataManagerImpl metadataManager = - (OmMetadataManagerImpl) omMetadataManager; + (OmMetadataManagerImpl) getOmMetadataManager(); SnapshotChainManager chainManager = metadataManager .getSnapshotChainManager(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java index ab2bac1bd0e..a746597288a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java @@ -17,17 +17,8 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.audit.AuditMessage; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -35,17 +26,14 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.io.File; import java.util.UUID; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; @@ -62,75 +50,19 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.framework; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** * Tests OMSnapshotRenameRequest class, which handles RenameSnapshot request. */ -public class TestOMSnapshotRenameRequest { - - @TempDir - private File anotherTempDir; - - private OzoneManager ozoneManager; - private OMMetrics omMetrics; - private OmMetadataManagerImpl omMetadataManager; - private BatchOperation batchOperation; - - private String volumeName; - private String bucketName; +public class TestOMSnapshotRenameRequest extends TestSnapshotRequestAndResponse { private String snapshotName1; private String snapshotName2; @BeforeEach public void setup() throws Exception { - ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, - anotherTempDir.getAbsolutePath()); - ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - anotherTempDir.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, - ozoneManager); - when(ozoneManager.getMetrics()).thenReturn(omMetrics); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - when(ozoneManager.isRatisEnabled()).thenReturn(true); - when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); - when(ozoneManager.isAdmin(any())).thenReturn(false); - when(ozoneManager.isOwner(any(), any())).thenReturn(false); - when(ozoneManager.getBucketOwner(any(), any(), - any(), any())).thenReturn("dummyBucketOwner"); - OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); - when(lvm.isAllowed(anyString())).thenReturn(true); - when(ozoneManager.getVersionManager()).thenReturn(lvm); - AuditLogger auditLogger = mock(AuditLogger.class); - when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); - doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); - batchOperation = omMetadataManager.getStore().initBatchOperation(); - when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); - OmSnapshotManager omSnapshotManager = new OmSnapshotManager(ozoneManager); - when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); - - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); snapshotName1 = UUID.randomUUID().toString(); snapshotName2 = UUID.randomUUID().toString(); - OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, - omMetadataManager); - } - - @AfterEach - public void stop() { - omMetrics.unRegister(); - framework().clearInlineMocks(); - if (batchOperation != null) { - batchOperation.close(); - } } @ValueSource(strings = { @@ -143,11 +75,11 @@ public void stop() { }) @ParameterizedTest public void testPreExecute(String toSnapshotName) throws Exception { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); String currentSnapshotName = "current"; - OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(volumeName, - bucketName, currentSnapshotName, toSnapshotName); + OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), + getBucketName(), currentSnapshotName, toSnapshotName); doPreExecute(omRequest); } @@ -167,10 +99,10 @@ public void testPreExecute(String toSnapshotName) throws Exception { }) @ParameterizedTest public void testPreExecuteFailure(String toSnapshotName) { - when(ozoneManager.isOwner(any(), any())).thenReturn(true); + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); String currentSnapshotName = "current"; - OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(volumeName, - bucketName, currentSnapshotName, toSnapshotName); + OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), + getBucketName(), currentSnapshotName, toSnapshotName); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); assertTrue(omException.getMessage().contains("Invalid snapshot name: " + toSnapshotName)); @@ -179,8 +111,8 @@ public void testPreExecuteFailure(String toSnapshotName) { @Test public void testPreExecuteBadOwner() { // Owner is not set for the request. - OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(volumeName, - bucketName, snapshotName1, snapshotName2); + OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName1, snapshotName2); OMException omException = assertThrows(OMException.class, () -> doPreExecute(omRequest)); @@ -190,39 +122,39 @@ public void testPreExecuteBadOwner() { @Test public void testValidateAndUpdateCache() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); - OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(volumeName, - bucketName, snapshotName1, snapshotName2); + when(getOzoneManager().isAdmin(any())).thenReturn(true); + OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName1, snapshotName2); OMSnapshotRenameRequest omSnapshotRenameRequest = doPreExecute(omRequest); - String key = getTableKey(volumeName, bucketName, snapshotName1); - String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + String key = getTableKey(getVolumeName(), getBucketName(), snapshotName1); + String bucketKey = getOmMetadataManager().getBucketKey(getVolumeName(), getBucketName()); // Add a 1000-byte key to the bucket OmKeyInfo key1 = addKey("key-testValidateAndUpdateCache", 12345L); addKeyToTable(key1); - OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( + OmBucketInfo omBucketInfo = getOmMetadataManager().getBucketTable().get( bucketKey); long bucketDataSize = key1.getDataSize(); long bucketUsedBytes = omBucketInfo.getUsedBytes(); assertEquals(key1.getReplicatedSize(), bucketUsedBytes); // Value in cache should be null as of now. - assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key)); // Add key to cache. - SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(volumeName, bucketName, + SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(getVolumeName(), getBucketName(), snapshotName1, UUID.randomUUID(), Time.now()); snapshotInfo.setReferencedSize(1000L); snapshotInfo.setReferencedReplicatedSize(3 * 1000L); assertEquals(SNAPSHOT_ACTIVE, snapshotInfo.getSnapshotStatus()); - omMetadataManager.getSnapshotInfoTable().addCacheEntry( + getOmMetadataManager().getSnapshotInfoTable().addCacheEntry( new CacheKey<>(key), CacheValue.get(1L, snapshotInfo)); // Run validateAndUpdateCache. OMClientResponse omClientResponse = - omSnapshotRenameRequest.validateAndUpdateCache(ozoneManager, 2L); + omSnapshotRenameRequest.validateAndUpdateCache(getOzoneManager(), 2L); assertNotNull(omClientResponse.getOMResponse()); @@ -244,56 +176,56 @@ public void testValidateAndUpdateCache() throws Exception { SnapshotInfo snapshotInfoOldProto = getFromProtobuf(snapshotInfoProto); - String key2 = getTableKey(volumeName, bucketName, snapshotName2); + String key2 = getTableKey(getVolumeName(), getBucketName(), snapshotName2); // Get value from cache SnapshotInfo snapshotInfoNewInCache = - omMetadataManager.getSnapshotInfoTable().get(key2); + getOmMetadataManager().getSnapshotInfoTable().get(key2); assertNotNull(snapshotInfoNewInCache); assertEquals(snapshotInfoOldProto, snapshotInfoNewInCache); assertEquals(snapshotInfo.getSnapshotId(), snapshotInfoNewInCache.getSnapshotId()); SnapshotInfo snapshotInfoOldInCache = - omMetadataManager.getSnapshotInfoTable().get(key); + getOmMetadataManager().getSnapshotInfoTable().get(key); assertNull(snapshotInfoOldInCache); } @Test public void testEntryExists() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); - String keyNameOld = getTableKey(volumeName, bucketName, snapshotName1); - String keyNameNew = getTableKey(volumeName, bucketName, snapshotName2); + String keyNameOld = getTableKey(getVolumeName(), getBucketName(), snapshotName1); + String keyNameNew = getTableKey(getVolumeName(), getBucketName(), snapshotName2); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameOld)); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameNew)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameOld)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameNew)); // First make sure we have two snapshots. OzoneManagerProtocolProtos.OMRequest createOmRequest = - createSnapshotRequest(volumeName, bucketName, snapshotName1); + createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); OMSnapshotCreateRequest omSnapshotCreateRequest = - TestOMSnapshotCreateRequest.doPreExecute(createOmRequest, ozoneManager); - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); + TestOMSnapshotCreateRequest.doPreExecute(createOmRequest, getOzoneManager()); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1); createOmRequest = - createSnapshotRequest(volumeName, bucketName, snapshotName2); + createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName2); omSnapshotCreateRequest = - TestOMSnapshotCreateRequest.doPreExecute(createOmRequest, ozoneManager); - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 2); + TestOMSnapshotCreateRequest.doPreExecute(createOmRequest, getOzoneManager()); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 2); - assertNotNull(omMetadataManager.getSnapshotInfoTable().get(keyNameOld)); - assertNotNull(omMetadataManager.getSnapshotInfoTable().get(keyNameNew)); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameOld)); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameNew)); // Now try renaming and get an error. OzoneManagerProtocolProtos.OMRequest omRequest = - renameSnapshotRequest(volumeName, bucketName, snapshotName1, snapshotName2); + renameSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1, snapshotName2); OMSnapshotRenameRequest omSnapshotRenameRequest = doPreExecute(omRequest); OMClientResponse omClientResponse = - omSnapshotRenameRequest.validateAndUpdateCache(ozoneManager, 3); + omSnapshotRenameRequest.validateAndUpdateCache(getOzoneManager(), 3); - assertNotNull(omMetadataManager.getSnapshotInfoTable().get(keyNameOld)); - assertNotNull(omMetadataManager.getSnapshotInfoTable().get(keyNameNew)); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameOld)); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameNew)); OzoneManagerProtocolProtos.OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse.getRenameSnapshotResponse()); @@ -303,24 +235,24 @@ public void testEntryExists() throws Exception { @Test public void testEntryNotFound() throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); + when(getOzoneManager().isAdmin(any())).thenReturn(true); - String keyNameOld = getTableKey(volumeName, bucketName, snapshotName1); - String keyNameNew = getTableKey(volumeName, bucketName, snapshotName2); + String keyNameOld = getTableKey(getVolumeName(), getBucketName(), snapshotName1); + String keyNameNew = getTableKey(getVolumeName(), getBucketName(), snapshotName2); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameOld)); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameNew)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameOld)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameNew)); // Now try renaming and get an error. OzoneManagerProtocolProtos.OMRequest omRequest = - renameSnapshotRequest(volumeName, bucketName, snapshotName1, snapshotName2); + renameSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1, snapshotName2); OMSnapshotRenameRequest omSnapshotRenameRequest = doPreExecute(omRequest); OMClientResponse omClientResponse = - omSnapshotRenameRequest.validateAndUpdateCache(ozoneManager, 3); + omSnapshotRenameRequest.validateAndUpdateCache(getOzoneManager(), 3); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameOld)); - assertNull(omMetadataManager.getSnapshotInfoTable().get(keyNameNew)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameOld)); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(keyNameNew)); OzoneManagerProtocolProtos.OMResponse omResponse = omClientResponse.getOMResponse(); assertNotNull(omResponse.getRenameSnapshotResponse()); @@ -330,7 +262,7 @@ public void testEntryNotFound() throws Exception { private OMSnapshotRenameRequest doPreExecute( OzoneManagerProtocolProtos.OMRequest originalRequest) throws Exception { - return doPreExecute(originalRequest, ozoneManager); + return doPreExecute(originalRequest, getOzoneManager()); } public static OMSnapshotRenameRequest doPreExecute( @@ -344,15 +276,15 @@ public static OMSnapshotRenameRequest doPreExecute( } private OmKeyInfo addKey(String keyName, long objectId) { - return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + return OMRequestTestUtils.createOmKeyInfo(getVolumeName(), getBucketName(), keyName, RatisReplicationConfig.getInstance(THREE)).setObjectID(objectId) .build(); } protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception { OMRequestTestUtils.addKeyToTable(false, true, keyInfo, 0, 0L, - omMetadataManager); - return omMetadataManager.getOzoneKey(keyInfo.getVolumeName(), + getOmMetadataManager()); + return getOmMetadataManager().getOzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java index b5bfc2714b0..380922f9e22 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java @@ -18,32 +18,23 @@ */ package org.apache.hadoop.ozone.om.request.snapshot; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotSetPropertyResponse; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.io.TempDir; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -60,37 +51,13 @@ * Tests TestOMSnapshotSetPropertyRequest * TestOMSnapshotSetPropertyResponse class. */ -public class TestOMSnapshotSetPropertyRequestAndResponse { - private BatchOperation batchOperation; - private OzoneManager ozoneManager; - private OMMetadataManager omMetadataManager; - private OMMetrics omMetrics; - private String volumeName; - private String bucketName; +public class TestOMSnapshotSetPropertyRequestAndResponse extends TestSnapshotRequestAndResponse { private String snapName; private long exclusiveSize; private long exclusiveSizeAfterRepl; @BeforeEach - void setup(@TempDir File testDir) throws Exception { - omMetrics = OMMetrics.create(); - ozoneManager = mock(OzoneManager.class); - OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); - when(lvm.isAllowed(anyString())).thenReturn(true); - when(ozoneManager.getVersionManager()).thenReturn(lvm); - when(ozoneManager.isRatisEnabled()).thenReturn(true); - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, - testDir.getAbsolutePath()); - ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - testDir.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, - ozoneManager); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - when(ozoneManager.getMetrics()).thenReturn(omMetrics); - - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); + void setup() { snapName = UUID.randomUUID().toString(); exclusiveSize = 2000L; exclusiveSizeAfterRepl = 6000L; @@ -98,11 +65,11 @@ void setup(@TempDir File testDir) throws Exception { @Test public void testValidateAndUpdateCache() throws IOException { - long initialSnapshotSetPropertyCount = omMetrics.getNumSnapshotSetProperties(); - long initialSnapshotSetPropertyFailCount = omMetrics.getNumSnapshotSetPropertyFails(); + long initialSnapshotSetPropertyCount = getOmMetrics().getNumSnapshotSetProperties(); + long initialSnapshotSetPropertyFailCount = getOmMetrics().getNumSnapshotSetPropertyFails(); createSnapshotDataForTest(); - assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); + assertFalse(getOmMetadataManager().getSnapshotInfoTable().isEmpty()); List snapshotUpdateSizeRequests = createSnapshotUpdateSizeRequest(); @@ -111,28 +78,27 @@ public void testValidateAndUpdateCache() throws IOException { OMSnapshotSetPropertyRequest omSnapshotSetPropertyRequest = new OMSnapshotSetPropertyRequest(request); OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest - .preExecute(ozoneManager); + .preExecute(getOzoneManager()); omSnapshotSetPropertyRequest = new OMSnapshotSetPropertyRequest(modifiedOmRequest); // Validate and Update Cache OMSnapshotSetPropertyResponse omSnapshotSetPropertyResponse = (OMSnapshotSetPropertyResponse) omSnapshotSetPropertyRequest - .validateAndUpdateCache(ozoneManager, 200L); + .validateAndUpdateCache(getOzoneManager(), 200L); // Commit to DB. - batchOperation = omMetadataManager.getStore().initBatchOperation(); - omSnapshotSetPropertyResponse.checkAndUpdateDB(omMetadataManager, - batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + omSnapshotSetPropertyResponse.checkAndUpdateDB(getOmMetadataManager(), + getBatchOperation()); + getOmMetadataManager().getStore().commitBatchOperation(getBatchOperation()); } assertEquals(initialSnapshotSetPropertyCount + snapshotUpdateSizeRequests.size(), - omMetrics.getNumSnapshotSetProperties()); - assertEquals(initialSnapshotSetPropertyFailCount, omMetrics.getNumSnapshotSetPropertyFails()); + getOmMetrics().getNumSnapshotSetProperties()); + assertEquals(initialSnapshotSetPropertyFailCount, getOmMetrics().getNumSnapshotSetPropertyFails()); // Check if the exclusive size is set. try (TableIterator> - iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { + iterator = getOmMetadataManager().getSnapshotInfoTable().iterator()) { while (iterator.hasNext()) { Table.KeyValue snapshotEntry = iterator.next(); assertCacheValues(snapshotEntry.getKey()); @@ -149,11 +115,11 @@ public void testValidateAndUpdateCache() throws IOException { */ @Test public void testValidateAndUpdateCacheFailure() throws IOException { - long initialSnapshotSetPropertyCount = omMetrics.getNumSnapshotSetProperties(); - long initialSnapshotSetPropertyFailCount = omMetrics.getNumSnapshotSetPropertyFails(); + long initialSnapshotSetPropertyCount = getOmMetrics().getNumSnapshotSetProperties(); + long initialSnapshotSetPropertyFailCount = getOmMetrics().getNumSnapshotSetPropertyFails(); createSnapshotDataForTest(); - assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); + assertFalse(getOmMetadataManager().getSnapshotInfoTable().isEmpty()); List snapshotUpdateSizeRequests = createSnapshotUpdateSizeRequest(); OmMetadataManagerImpl mockedMetadataManager = mock(OmMetadataManagerImpl.class); @@ -161,27 +127,27 @@ public void testValidateAndUpdateCacheFailure() throws IOException { when(mockedSnapshotInfoTable.get(anyString())).thenThrow(new IOException("Injected fault error.")); when(mockedMetadataManager.getSnapshotInfoTable()).thenReturn(mockedSnapshotInfoTable); - when(ozoneManager.getMetadataManager()).thenReturn(mockedMetadataManager); + when(getOzoneManager().getMetadataManager()).thenReturn(mockedMetadataManager); for (OMRequest omRequest: snapshotUpdateSizeRequests) { OMSnapshotSetPropertyRequest omSnapshotSetPropertyRequest = new OMSnapshotSetPropertyRequest(omRequest); - OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest.preExecute(ozoneManager); + OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest.preExecute(getOzoneManager()); omSnapshotSetPropertyRequest = new OMSnapshotSetPropertyRequest(modifiedOmRequest); // Validate and Update Cache OMSnapshotSetPropertyResponse omSnapshotSetPropertyResponse = (OMSnapshotSetPropertyResponse) - omSnapshotSetPropertyRequest.validateAndUpdateCache(ozoneManager, 200L); + omSnapshotSetPropertyRequest.validateAndUpdateCache(getOzoneManager(), 200L); assertEquals(INTERNAL_ERROR, omSnapshotSetPropertyResponse.getOMResponse().getStatus()); } - assertEquals(initialSnapshotSetPropertyCount, omMetrics.getNumSnapshotSetProperties()); + assertEquals(initialSnapshotSetPropertyCount, getOmMetrics().getNumSnapshotSetProperties()); assertEquals(initialSnapshotSetPropertyFailCount + snapshotUpdateSizeRequests.size(), - omMetrics.getNumSnapshotSetPropertyFails()); + getOmMetrics().getNumSnapshotSetPropertyFails()); } private void assertCacheValues(String dbKey) { - CacheValue cacheValue = omMetadataManager + CacheValue cacheValue = getOmMetadataManager() .getSnapshotInfoTable() .getCacheValue(new CacheKey<>(dbKey)); assertEquals(exclusiveSize, cacheValue.getCacheValue().getExclusiveSize()); @@ -193,7 +159,7 @@ private List createSnapshotUpdateSizeRequest() throws IOException { List omRequests = new ArrayList<>(); try (TableIterator> - iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { + iterator = getOmMetadataManager().getSnapshotInfoTable().iterator()) { while (iterator.hasNext()) { String snapDbKey = iterator.next().getKey(); SnapshotSize snapshotSize = SnapshotSize.newBuilder() @@ -220,8 +186,8 @@ private List createSnapshotUpdateSizeRequest() private void createSnapshotDataForTest() throws IOException { // Create 10 Snapshots for (int i = 0; i < 10; i++) { - OMRequestTestUtils.addSnapshotToTableCache(volumeName, bucketName, - snapName + i, omMetadataManager); + OMRequestTestUtils.addSnapshotToTableCache(getVolumeName(), getBucketName(), + snapName + i, getOmMetadataManager()); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotMoveTableKeysResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotMoveTableKeysResponse.java new file mode 100644 index 00000000000..d2e2d94ec73 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotMoveTableKeysResponse.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +/** + * Test class to test OMSnapshotMoveTableKeysResponse. + */ +public class TestOMSnapshotMoveTableKeysResponse extends TestSnapshotRequestAndResponse { + + private String snapshotName1; + private String snapshotName2; + private SnapshotInfo snapshotInfo1; + private SnapshotInfo snapshotInfo2; + + @BeforeEach + public void setup() throws Exception { + snapshotName1 = UUID.randomUUID().toString(); + snapshotName2 = UUID.randomUUID().toString(); + } + + public TestOMSnapshotMoveTableKeysResponse() { + super(true); + } + + private void createSnapshots(boolean createSecondSnapshot) throws Exception { + addDataToTable(getOmMetadataManager().getSnapshotRenamedTable(), getRenameKeys(getVolumeName(), getBucketName(), 0, + 10, snapshotName1)); + addDataToTable(getOmMetadataManager().getDeletedTable(), getDeletedKeys(getVolumeName(), getBucketName(), 0, + 10, 10, 0).stream() + .map(pair -> Pair.of(pair.getKey(), new RepeatedOmKeyInfo(pair.getRight()))) + .collect(Collectors.toList())); + addDataToTable(getOmMetadataManager().getDeletedDirTable(), + getDeletedDirKeys(getVolumeName(), getBucketName(), 0, 10, 1).stream() + .map(pair -> Pair.of(pair.getKey(), pair.getRight().get(0))).collect(Collectors.toList())); + createSnapshotCheckpoint(getVolumeName(), getBucketName(), snapshotName1); + snapshotInfo1 = SnapshotUtils.getSnapshotInfo(getOzoneManager(), getVolumeName(), getBucketName(), snapshotName1); + addDataToTable(getOmMetadataManager().getSnapshotRenamedTable(), getRenameKeys(getVolumeName(), getBucketName(), 5, + 15, snapshotName2)); + addDataToTable(getOmMetadataManager().getDeletedTable(), getDeletedKeys(getVolumeName(), getBucketName(), 5, + 8, 10, 10).stream() + .map(pair -> Pair.of(pair.getKey(), new RepeatedOmKeyInfo(pair.getRight()))) + .collect(Collectors.toList())); + addDataToTable(getOmMetadataManager().getDeletedTable(), getDeletedKeys(getVolumeName(), getBucketName(), 8, + 15, 10, 0).stream() + .map(pair -> Pair.of(pair.getKey(), new RepeatedOmKeyInfo(pair.getRight()))) + .collect(Collectors.toList())); + addDataToTable(getOmMetadataManager().getDeletedDirTable(), + getDeletedDirKeys(getVolumeName(), getBucketName(), 5, 15, 1).stream() + .map(pair -> Pair.of(pair.getKey(), pair.getRight().get(0))).collect(Collectors.toList())); + if (createSecondSnapshot) { + createSnapshotCheckpoint(getVolumeName(), getBucketName(), snapshotName2); + snapshotInfo2 = SnapshotUtils.getSnapshotInfo(getOzoneManager(), getVolumeName(), getBucketName(), snapshotName2); + } + } + + private void addDataToTable(Table table, List> vals) throws IOException { + for (Pair pair : vals) { + table.put(pair.getKey(), pair.getValue()); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testMoveTableKeysToNextSnapshot(boolean nextSnapshotExists) throws Exception { + createSnapshots(nextSnapshotExists); + + try (ReferenceCounted snapshot1 = getOmSnapshotManager().getSnapshot(getVolumeName(), getBucketName(), + snapshotName1); + ReferenceCounted snapshot2 = nextSnapshotExists ? getOmSnapshotManager().getSnapshot( + getVolumeName(), getBucketName(), snapshotName2) : null) { + OmSnapshot snapshot = snapshot1.get(); + List deletedTable = new ArrayList<>(); + List deletedDirTable = new ArrayList<>(); + List renamedTable = new ArrayList<>(); + Map renameEntries = new HashMap<>(); + snapshot.getMetadataManager().getDeletedTable().iterator() + .forEachRemaining(entry -> { + try { + deletedTable.add(OzoneManagerProtocolProtos.SnapshotMoveKeyInfos.newBuilder().setKey(entry.getKey()) + .addAllKeyInfos(entry.getValue().getOmKeyInfoList().stream().map(omKeyInfo -> omKeyInfo.getProtobuf( + ClientVersion.CURRENT_VERSION)).collect(Collectors.toList())).build()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + snapshot.getMetadataManager().getDeletedDirTable().iterator() + .forEachRemaining(entry -> { + try { + deletedDirTable.add(OzoneManagerProtocolProtos.SnapshotMoveKeyInfos.newBuilder().setKey(entry.getKey()) + .addKeyInfos(entry.getValue().getProtobuf(ClientVersion.CURRENT_VERSION)).build()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + snapshot.getMetadataManager().getSnapshotRenamedTable().iterator().forEachRemaining(entry -> { + try { + renamedTable.add(HddsProtos.KeyValue.newBuilder().setKey(entry.getKey()).setValue(entry.getValue()).build()); + renameEntries.put(entry.getKey(), entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + OMSnapshotMoveTableKeysResponse response = new OMSnapshotMoveTableKeysResponse( + OzoneManagerProtocolProtos.OMResponse.newBuilder().setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotMoveTableKeys).build(), + snapshotInfo1, nextSnapshotExists ? snapshotInfo2 : null, deletedTable, deletedDirTable, renamedTable); + try (BatchOperation batchOperation = getOmMetadataManager().getStore().initBatchOperation()) { + response.addToDBBatch(getOmMetadataManager(), batchOperation); + getOmMetadataManager().getStore().commitBatchOperation(batchOperation); + } + Assertions.assertTrue(snapshot.getMetadataManager().getDeletedTable().isEmpty()); + Assertions.assertTrue(snapshot.getMetadataManager().getDeletedDirTable().isEmpty()); + Assertions.assertTrue(snapshot.getMetadataManager().getSnapshotRenamedTable().isEmpty()); + OMMetadataManager nextMetadataManager = + nextSnapshotExists ? snapshot2.get().getMetadataManager() : getOmMetadataManager(); + AtomicInteger count = new AtomicInteger(); + nextMetadataManager.getDeletedTable().iterator().forEachRemaining(entry -> { + count.getAndIncrement(); + try { + int maxCount = count.get() >= 6 && count.get() <= 8 ? 20 : 10; + Assertions.assertEquals(maxCount, entry.getValue().getOmKeyInfoList().size()); + List versions = entry.getValue().getOmKeyInfoList().stream().map(OmKeyInfo::getKeyLocationVersions) + .map(omKeyInfo -> omKeyInfo.get(0).getVersion()).collect(Collectors.toList()); + List expectedVersions = new ArrayList<>(); + if (maxCount == 20) { + expectedVersions.addAll(LongStream.range(10, 20).boxed().collect(Collectors.toList())); + } + expectedVersions.addAll(LongStream.range(0, 10).boxed().collect(Collectors.toList())); + Assertions.assertEquals(expectedVersions, versions); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + Assertions.assertEquals(15, count.get()); + count.set(0); + + nextMetadataManager.getDeletedDirTable().iterator().forEachRemaining(entry -> count.getAndIncrement()); + Assertions.assertEquals(15, count.get()); + count.set(0); + nextMetadataManager.getSnapshotRenamedTable().iterator().forEachRemaining(entry -> { + try { + String expectedValue = renameEntries.getOrDefault(entry.getKey(), entry.getValue()); + Assertions.assertEquals(expectedValue, entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + count.getAndIncrement(); + }); + Assertions.assertEquals(15, count.get()); + } + + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index 3948f4fab80..e04891da83a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -20,7 +20,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.KeyManagerImpl; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -56,25 +57,26 @@ public class TestSnapshotDeletingService { private SnapshotChainManager chainManager; @Mock private OmMetadataManagerImpl omMetadataManager; - @Mock - private ScmBlockLocationProtocol scmClient; private final OzoneConfiguration conf = new OzoneConfiguration();; private final long sdsRunInterval = Duration.ofMillis(1000).toMillis(); private final long sdsServiceTimeout = Duration.ofSeconds(10).toMillis(); - private static Stream testCasesForIgnoreSnapshotGc() { - SnapshotInfo filteredSnapshot = SnapshotInfo.newBuilder().setSstFiltered(true).setName("snap1").build(); - SnapshotInfo unFilteredSnapshot = SnapshotInfo.newBuilder().setSstFiltered(false).setName("snap1").build(); + private static Stream testCasesForIgnoreSnapshotGc() throws IOException { + SnapshotInfo flushedSnapshot = SnapshotInfo.newBuilder().setSstFiltered(true) + .setLastTransactionInfo(TransactionInfo.valueOf(1, 1).toByteString()) + .setName("snap1").build(); + SnapshotInfo unFlushedSnapshot = SnapshotInfo.newBuilder().setSstFiltered(false).setName("snap1") + .setLastTransactionInfo(TransactionInfo.valueOf(0, 0).toByteString()).build(); return Stream.of( - Arguments.of(filteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), - Arguments.of(filteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), - Arguments.of(unFilteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), - Arguments.of(unFilteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), - Arguments.of(filteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), - Arguments.of(unFilteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), - Arguments.of(unFilteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), - Arguments.of(filteredSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true)); + Arguments.of(flushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), + Arguments.of(flushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), + Arguments.of(unFlushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), + Arguments.of(unFlushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), + Arguments.of(flushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), + Arguments.of(unFlushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED, false), + Arguments.of(unFlushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true), + Arguments.of(flushedSnapshot, SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE, true)); } @ParameterizedTest @@ -87,9 +89,15 @@ public void testProcessSnapshotLogicInSDS(SnapshotInfo snapshotInfo, Mockito.when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); Mockito.when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); Mockito.when(ozoneManager.getConfiguration()).thenReturn(conf); + if (status == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) { + Table transactionInfoTable = Mockito.mock(Table.class); + Mockito.when(omMetadataManager.getTransactionInfoTable()).thenReturn(transactionInfoTable); + Mockito.when(transactionInfoTable.getSkipCache(Mockito.anyString())) + .thenReturn(TransactionInfo.valueOf(1, 1)); + } SnapshotDeletingService snapshotDeletingService = - new SnapshotDeletingService(sdsRunInterval, sdsServiceTimeout, ozoneManager, scmClient); + new SnapshotDeletingService(sdsRunInterval, sdsServiceTimeout, ozoneManager); snapshotInfo.setSnapshotStatus(status); assertEquals(expectedOutcome, snapshotDeletingService.shouldIgnoreSnapshot(snapshotInfo)); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java index c5ae809718e..f49bfc33976 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; import com.google.common.collect.ImmutableMap; +import org.apache.commons.compress.utils.Lists; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -38,6 +39,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -168,6 +170,7 @@ public void testAddSnapshot() throws Exception { } assertEquals(snapshotID3, chainManager.getLatestGlobalSnapshotId()); + assertEquals(snapshotID1, chainManager.getOldestGlobalSnapshotId()); assertEquals(snapshotID3, chainManager.getLatestPathSnapshotId( String.join("/", "vol1", "bucket1"))); @@ -285,6 +288,7 @@ public void testChainFromLoadFromTable(boolean increasingTIme) assertFalse(chainManager.isSnapshotChainCorrupted()); // check if snapshots loaded correctly from snapshotInfoTable assertEquals(snapshotID2, chainManager.getLatestGlobalSnapshotId()); + assertEquals(snapshotID1, chainManager.getOldestGlobalSnapshotId()); assertEquals(snapshotID2, chainManager.nextGlobalSnapshot(snapshotID1)); assertEquals(snapshotID1, chainManager.previousPathSnapshot(String .join("/", "vol1", "bucket1"), snapshotID2)); @@ -305,6 +309,34 @@ public void testChainFromLoadFromTable(boolean increasingTIme) () -> chainManager.nextGlobalSnapshot(snapshotID1)); } + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 5, 10}) + public void testSnapshotChainIterator(int numberOfSnapshots) throws IOException { + Table snapshotInfo = omMetadataManager.getSnapshotInfoTable(); + List snapshotInfoList = new ArrayList<>(); + + UUID prevSnapshotID = null; + long time = System.currentTimeMillis(); + for (int i = 0; i < numberOfSnapshots; i++) { + UUID snapshotID = UUID.randomUUID(); + SnapshotInfo snapInfo = createSnapshotInfo(snapshotID, prevSnapshotID, + prevSnapshotID, time++); + snapshotInfo.put(snapshotID.toString(), snapInfo); + prevSnapshotID = snapshotID; + snapshotInfoList.add(snapInfo); + } + chainManager = new SnapshotChainManager(omMetadataManager); + assertFalse(chainManager.isSnapshotChainCorrupted()); + List reverseChain = Lists.newArrayList(chainManager.iterator(true)); + Collections.reverse(reverseChain); + List forwardChain = Lists.newArrayList(chainManager.iterator(false)); + List expectedChain = snapshotInfoList.stream().map(SnapshotInfo::getSnapshotId).collect(Collectors.toList()); + assertEquals(expectedChain, reverseChain); + assertEquals(expectedChain, forwardChain); + assertEquals(forwardChain, reverseChain); + + } + private static Stream invalidSnapshotChain() { List nodes = IntStream.range(0, 5) .mapToObj(i -> UUID.randomUUID()) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java new file mode 100644 index 00000000000..e60e23de22a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; +import org.apache.hadoop.ozone.om.request.snapshot.TestOMSnapshotCreateRequest; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.framework; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Base class to test snapshot functionalities. + */ +public class TestSnapshotRequestAndResponse { + @TempDir + private File testDir; + + private OzoneManager ozoneManager; + private OMMetrics omMetrics; + private OmMetadataManagerImpl omMetadataManager; + private BatchOperation batchOperation; + private OmSnapshotManager omSnapshotManager; + + private String volumeName; + private String bucketName; + private boolean isAdmin; + + public BatchOperation getBatchOperation() { + return batchOperation; + } + + public String getBucketName() { + return bucketName; + } + + public boolean isAdmin() { + return isAdmin; + } + + public OmMetadataManagerImpl getOmMetadataManager() { + return omMetadataManager; + } + + public OMMetrics getOmMetrics() { + return omMetrics; + } + + public OmSnapshotManager getOmSnapshotManager() { + return omSnapshotManager; + } + + public OzoneManager getOzoneManager() { + return ozoneManager; + } + + public File getTestDir() { + return testDir; + } + + public String getVolumeName() { + return volumeName; + } + + protected TestSnapshotRequestAndResponse() { + this.isAdmin = false; + } + + protected TestSnapshotRequestAndResponse(boolean isAdmin) { + this.isAdmin = isAdmin; + } + + @BeforeEach + public void baseSetup() throws Exception { + ozoneManager = mock(OzoneManager.class); + omMetrics = OMMetrics.create(); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + testDir.getAbsolutePath()); + ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, + testDir.getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, + ozoneManager); + when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); + when(ozoneManager.getMetrics()).thenReturn(omMetrics); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + when(ozoneManager.isRatisEnabled()).thenReturn(true); + when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); + when(ozoneManager.isAdmin(any())).thenReturn(isAdmin); + when(ozoneManager.isOwner(any(), any())).thenReturn(false); + when(ozoneManager.getBucketOwner(any(), any(), + any(), any())).thenReturn("dummyBucketOwner"); + IAccessAuthorizer accessAuthorizer = mock(IAccessAuthorizer.class); + when(ozoneManager.getAccessAuthorizer()).thenReturn(accessAuthorizer); + when(accessAuthorizer.isNative()).thenReturn(false); + OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); + when(lvm.isAllowed(anyString())).thenReturn(true); + when(ozoneManager.getVersionManager()).thenReturn(lvm); + AuditLogger auditLogger = mock(AuditLogger.class); + when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); + doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); + batchOperation = omMetadataManager.getStore().initBatchOperation(); + + volumeName = UUID.randomUUID().toString(); + bucketName = UUID.randomUUID().toString(); + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + omSnapshotManager = new OmSnapshotManager(ozoneManager); + when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); + } + + @AfterEach + public void stop() { + omMetrics.unRegister(); + framework().clearInlineMocks(); + if (batchOperation != null) { + batchOperation.close(); + } + } + + protected Path createSnapshotCheckpoint(String volume, String bucket, String snapshotName) throws Exception { + OzoneManagerProtocolProtos.OMRequest omRequest = OMRequestTestUtils + .createSnapshotRequest(volume, bucket, snapshotName); + // Pre-Execute OMSnapshotCreateRequest. + OMSnapshotCreateRequest omSnapshotCreateRequest = + TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager); + + // validateAndUpdateCache OMSnapshotCreateResponse. + OMSnapshotCreateResponse omClientResponse = (OMSnapshotCreateResponse) + omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1); + // Add to batch and commit to DB. + try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { + omClientResponse.addToDBBatch(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + } + + String key = SnapshotInfo.getTableKey(volume, bucket, snapshotName); + SnapshotInfo snapshotInfo = + omMetadataManager.getSnapshotInfoTable().get(key); + assertNotNull(snapshotInfo); + + RDBStore store = (RDBStore) omMetadataManager.getStore(); + String checkpointPrefix = store.getDbLocation().getName(); + Path snapshotDirPath = Paths.get(store.getSnapshotsParentDir(), + checkpointPrefix + snapshotInfo.getCheckpointDir()); + // Check the DB is still there + assertTrue(Files.exists(snapshotDirPath)); + return snapshotDirPath; + } + + protected List>> getDeletedKeys(String volume, String bucket, + int startRange, int endRange, + int numberOfKeys, + int minVersion) { + return IntStream.range(startRange, endRange).boxed() + .map(i -> Pair.of(omMetadataManager.getOzoneDeletePathKey(i, + omMetadataManager.getOzoneKey(volume, bucket, "key" + String.format("%010d", i))), + IntStream.range(0, numberOfKeys).boxed().map(cnt -> createOmKeyInfo(volume, bucket, "key" + i, + ReplicationConfig.getDefault(ozoneManager.getConfiguration()), + new OmKeyLocationInfoGroup(minVersion + cnt, new ArrayList<>(), false)) + .setCreationTime(0).setModificationTime(0).build()) + .collect(Collectors.toList()))) + .collect(Collectors.toList()); + } + + protected List> getRenameKeys(String volume, String bucket, + int startRange, int endRange, + String renameKeyPrefix) { + return IntStream.range(startRange, endRange).boxed() + .map(i -> { + try { + return Pair.of(omMetadataManager.getRenameKey(volume, bucket, i), + omMetadataManager.getOzoneKeyFSO(volume, bucket, renameKeyPrefix + i)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); + } + + protected List>> getDeletedDirKeys(String volume, String bucket, + int startRange, int endRange, int numberOfKeys) { + return IntStream.range(startRange, endRange).boxed() + .map(i -> { + try { + return Pair.of(omMetadataManager.getOzoneDeletePathKey(i, + omMetadataManager.getOzoneKeyFSO(volume, bucket, "1/key" + i)), + IntStream.range(0, numberOfKeys).boxed().map(cnt -> createOmKeyInfo(volume, bucket, "key" + i, + ReplicationConfig.getDefault(ozoneManager.getConfiguration())).build()) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + +}