From c8724ffd7d743907b6f93f3b44f3b0fd5ee83531 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 12 Nov 2024 12:29:19 +0530 Subject: [PATCH] Added repair and debug options, exclude buckets with snapshots --- .../hadoop/fs/ozone/TestFSORepairTool.java | 338 ++++++++++-------- .../hadoop/ozone/repair/om/FSORepairCLI.java | 19 +- .../hadoop/ozone/repair/om/FSORepairTool.java | 80 ++++- 3 files changed, 275 insertions(+), 162 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestFSORepairTool.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestFSORepairTool.java index 004c2e8e2497..cd8ea84e163b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestFSORepairTool.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestFSORepairTool.java @@ -41,9 +41,9 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.repair.om.FSORepairTool; import org.apache.hadoop.ozone.shell.OzoneShell; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -54,18 +54,18 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.TimeUnit; import static java.lang.System.err; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; +import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -80,23 +80,22 @@ public class TestFSORepairTool { private static FileSystem fs; private static OzoneClient client; private static OzoneConfiguration conf = null; - + private static FSORepairTool tool; @BeforeAll public static void init() throws Exception { // Set configs. conf = new OzoneConfiguration(); // deletion services will be triggered manually. - conf.setTimeDuration(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, - 1_000_000, TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1_000_000, - TimeUnit.SECONDS); - conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 10); - conf.setInt(OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK, 10); + conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 2000); + conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5); + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, + TimeUnit.MILLISECONDS); + conf.setInt(OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK, 20); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true); // Since delete services use RocksDB iterators, make sure the double // buffer is flushed between runs. - conf.setInt(OMConfigKeys.OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT, 1); + conf.setInt(OMConfigKeys.OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT, 2); // Build cluster. cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf) @@ -130,6 +129,7 @@ public void cleanNamespace() throws Exception { assertEquals(0, exitC); } + cluster.getOzoneManager().prepareOzoneManager(120L, 5L); runDeletes(); assertFileAndDirTablesEmpty(); } @@ -151,6 +151,7 @@ private int execute(GenericCli shell, String[] args) { cmd.setExecutionExceptionHandler(exceptionHandler); return cmd.execute(argsWithHAConf); } + private String getSetConfStringFromConf(String key) { return String.format("--set=%s=%s", key, conf.get(key)); } @@ -212,12 +213,12 @@ public static void teardown() { @Test public void testConnectedTreeOneBucket() throws Exception { - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report expectedReport = buildConnectedTree("vol1", "bucket1"); + FSORepairTool.Report expectedReport = buildConnectedTree("vol1", "bucket1"); // Test the connected tree in debug mode. - FSORepairTool fsoTool = new FSORepairTool(getOmDB(), - getOmDBLocation(), true, null, null); - FSORepairTool.Report debugReport = fsoTool.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), true, false, null, null); + FSORepairTool.Report debugReport = tool.run(); Assertions.assertEquals(expectedReport, debugReport); assertConnectedTreeReadable("vol1", "bucket1"); @@ -225,9 +226,9 @@ public void testConnectedTreeOneBucket() throws Exception { // Running again in repair mode should give same results since the tree // is connected. - fsoTool = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report repairReport = fsoTool.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report repairReport = tool.run(); Assertions.assertEquals(expectedReport, repairReport); assertConnectedTreeReadable("vol1", "bucket1"); @@ -240,29 +241,103 @@ public void testReportedDataSize() throws Exception { FSORepairTool.Report report2 = buildConnectedTree("vol1", "bucket2", 10); FSORepairTool.Report expectedReport = new FSORepairTool.Report(report1, report2); - FSORepairTool - repair = new FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - FSORepairTool.Report debugReport = repair.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report debugReport = tool.run(); Assertions.assertEquals(expectedReport, debugReport); } + /** + * Test to verify how the tool processes the volume and bucket + * filters. + */ + @Test + public void testVolumeAndBucketFilter() throws Exception { + FSORepairTool.Report report1 = buildDisconnectedTree("vol1", "bucket1", 10); + FSORepairTool.Report report2 = buildConnectedTree("vol2", "bucket2", 10); + FSORepairTool.Report expectedReport1 = new FSORepairTool.Report(report1); + FSORepairTool.Report expectedReport2 = new FSORepairTool.Report(report2); + + // When volume filter is passed + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, "/vol1", null); + FSORepairTool.Report result1 = tool.run(); + Assertions.assertEquals(expectedReport1, result1); + + // When both volume and bucket filters are passed + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, "/vol2", "bucket2"); + FSORepairTool.Report result2 = tool.run(); + Assertions.assertEquals(expectedReport2, result2); + + PrintStream originalOut = System.out; + + // When a non-existent bucket filter is passed + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(outputStream)) { + System.setOut(ps); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, "/vol1", "bucket2"); + tool.run(); + String output = outputStream.toString(); + Assertions.assertTrue(output.contains("Bucket 'bucket2' does not exist in volume '/vol1'.")); + } finally { + System.setOut(originalOut); + } + + // When a non-existent volume filter is passed + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(outputStream)) { + System.setOut(ps); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, "/vol3", "bucket2"); + tool.run(); + String output = outputStream.toString(); + Assertions.assertTrue(output.contains("Volume '/vol3' does not exist.")); + } finally { + System.setOut(originalOut); + } + + // When bucket filter is passed without the volume filter. + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(outputStream)) { + System.setOut(ps); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, "bucket2"); + tool.run(); + String output = outputStream.toString(); + Assertions.assertTrue(output.contains("--bucket flag cannot be used without specifying --volume.")); + } finally { + System.setOut(originalOut); + } + + } + @Test public void testMultipleBucketsAndVolumes() throws Exception { + Table dirTable = + cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); + Table keyTable = + cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout()); FSORepairTool.Report report1 = buildConnectedTree("vol1", "bucket1"); FSORepairTool.Report report2 = buildDisconnectedTree("vol2", "bucket2"); - FSORepairTool.Report expectedAggregateReport = new org.apache.hadoop.ozone.repair.om.FSORepairTool.Report( + FSORepairTool.Report expectedAggregateReport = new FSORepairTool.Report( report1, report2); - org.apache.hadoop.ozone.repair.om.FSORepairTool - repair = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report generatedReport = repair.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report generatedReport = tool.run(); Assertions.assertEquals(generatedReport, expectedAggregateReport); assertConnectedTreeReadable("vol1", "bucket1"); assertDisconnectedTreePartiallyReadable("vol2", "bucket2"); - assertDisconnectedObjectsMarkedForDelete(1); + + // This assertion ensures that only specific directories and keys remain in the active tables, + // as the remaining entries are expected to be moved to the deleted tables by the background service. + // However, since the timing of the background deletion service is not predictable, + // assertions on the deleted tables themselves may lead to flaky tests. + assertEquals(4, countTableEntries(dirTable)); + assertEquals(5, countTableEntries(keyTable)); } /** @@ -271,6 +346,11 @@ public void testMultipleBucketsAndVolumes() throws Exception { */ @Test public void testDeleteOverwrite() throws Exception { + Table keyTable = + cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout()); + Table dirTable = + cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); + // Create files and dirs under dir1. To make sure they are added to the // delete table, the keys must have data. buildConnectedTree("vol1", "bucket1", 10); @@ -289,25 +369,28 @@ public void testDeleteOverwrite() throws Exception { ContractTestUtils.touch(fs, new Path("/vol1/bucket1/dir1/file2")); disconnectDirectory("dir1"); - org.apache.hadoop.ozone.repair.om.FSORepairTool - repair = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report generatedReport = repair.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report generatedReport = tool.run(); Assertions.assertEquals(1, generatedReport.getUnreachableDirs()); Assertions.assertEquals(3, generatedReport.getUnreachableFiles()); - assertDisconnectedObjectsMarkedForDelete(2); + // This assertion ensures that only specific directories and keys remain in the active tables, + // as the remaining entries are expected to be moved to the deleted tables by the background service. + // However, since the timing of the background deletion service is not predictable, + // assertions on the deleted tables themselves may lead to flaky tests. + assertEquals(1, countTableEntries(keyTable)); + assertEquals(1, countTableEntries(dirTable)); } @Test public void testEmptyFileTrees() throws Exception { // Run when there are no file trees. - org.apache.hadoop.ozone.repair.om.FSORepairTool - repair = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report generatedReport = repair.run(); - Assertions.assertEquals(generatedReport, new org.apache.hadoop.ozone.repair.om.FSORepairTool.Report()); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report generatedReport = tool.run(); + Assertions.assertEquals(generatedReport, new FSORepairTool.Report()); assertDeleteTablesEmpty(); // Create an empty volume and bucket. @@ -315,11 +398,10 @@ public void testEmptyFileTrees() throws Exception { fs.mkdirs(new Path("/vol2/bucket1")); // Run on an empty volume and bucket. - repair = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - generatedReport = repair.run(); - Assertions.assertEquals(generatedReport, new org.apache.hadoop.ozone.repair.om.FSORepairTool.Report()); - assertDeleteTablesEmpty(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + generatedReport = tool.run(); + Assertions.assertEquals(generatedReport, new FSORepairTool.Report()); } @Test @@ -349,15 +431,14 @@ public void testNonFSOBucketsSkipped() throws Exception { legacyStream.close(); // Add an FSO bucket with data. - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report connectReport = buildConnectedTree("vol1", "fso" + - "-bucket"); + FSORepairTool.Report connectReport = + buildConnectedTree("vol1", "fso-bucket"); // Even in repair mode there should be no action. legacy and obs buckets // will be skipped and FSO tree is connected. - org.apache.hadoop.ozone.repair.om.FSORepairTool - repair = new org.apache.hadoop.ozone.repair.om.FSORepairTool(getOmDB(), - getOmDBLocation(), false, null, null); - org.apache.hadoop.ozone.repair.om.FSORepairTool.Report generatedReport = repair.run(); + tool = new FSORepairTool(getOmDB(), + getOmDBLocation(), false, true, null, null); + FSORepairTool.Report generatedReport = tool.run(); Assertions.assertEquals(connectReport, generatedReport); assertConnectedTreeReadable("vol1", "fso-bucket"); @@ -372,7 +453,7 @@ public void testNonFSOBucketsSkipped() throws Exception { } - private org.apache.hadoop.ozone.repair.om.FSORepairTool.Report buildConnectedTree(String volume, String bucket) + private FSORepairTool.Report buildConnectedTree(String volume, String bucket) throws Exception { return buildConnectedTree(volume, bucket, 0); } @@ -380,8 +461,7 @@ private org.apache.hadoop.ozone.repair.om.FSORepairTool.Report buildConnectedTre /** * Creates a tree with 3 reachable directories and 4 reachable files. */ - private org.apache.hadoop.ozone.repair.om.FSORepairTool.Report buildConnectedTree(String volume, String bucket, - int fileSize) + private FSORepairTool.Report buildConnectedTree(String volume, String bucket, int fileSize) throws Exception { Path bucketPath = new Path("/" + volume + "/" + bucket); Path dir1 = new Path(bucketPath, "dir1"); @@ -445,17 +525,17 @@ private void assertConnectedTreeReadable(String volume, String bucket) Assertions.assertTrue(fs.exists(file4)); } - private org.apache.hadoop.ozone.repair.om.FSORepairTool.Report buildDisconnectedTree(String volume, String bucket) + private FSORepairTool.Report buildDisconnectedTree(String volume, String bucket) throws Exception { return buildDisconnectedTree(volume, bucket, 0); } /** - * Creates a tree with 2 reachable directories, 1 reachable file, 1 + * Creates a tree with 1 reachable directory, 1 reachable file, 1 * unreachable directory, and 3 unreachable files. */ - private org.apache.hadoop.ozone.repair.om.FSORepairTool.Report buildDisconnectedTree(String volume, String bucket, - int fileSize) throws Exception { + private FSORepairTool.Report buildDisconnectedTree(String volume, String bucket, + int fileSize) throws Exception { buildConnectedTree(volume, bucket, fileSize); // Manually remove dir1. This should disconnect 3 of the files and 1 of @@ -516,91 +596,61 @@ private void assertDisconnectedTreePartiallyReadable( Assertions.assertTrue(fs.exists(file4)); } - /** - * Checks that the disconnected tree's unreachable objects are correctly - * moved to the delete table. If the tree was written and deleted multiple - * times, it makes sure the delete entries with the same name are preserved. - */ - private void assertDisconnectedObjectsMarkedForDelete(int numWrites) - throws Exception { - - Map pendingDeleteDirCounts = new HashMap<>(); - - // Check deleted directory table. + private void assertDeleteTablesEmpty() throws Exception { OzoneManager leader = cluster.getOMLeader(); - Table deletedDirTable = - leader.getMetadataManager().getDeletedDirTable(); - try (TableIterator> iterator = - deletedDirTable.iterator()) { - while (iterator.hasNext()) { - Table.KeyValue entry = iterator.next(); - String key = entry.getKey(); - OmKeyInfo value = entry.getValue(); - - String dirName = key.split("/")[4]; - LOG.info("In deletedDirTable, extracting directory name {} from DB " + - "key {}", dirName, key); - // Check that the correct dir info was added. - // FSO delete path will fill in the whole path to the key in the - // proto when it is deleted. Once the tree is disconnected that can't - // be done, so just make sure the dirName contained in the key name - // somewhere. - Assertions.assertTrue(value.getKeyName().contains(dirName)); - - int count = pendingDeleteDirCounts.getOrDefault(dirName, 0); - pendingDeleteDirCounts.put(dirName, count + 1); + GenericTestUtils.waitFor(() -> { + try { + return leader.getMetadataManager().getDeletedDirTable().isEmpty(); + } catch (Exception e) { + LOG.error("DB failure!", e); + fail("DB failure!"); + return false; } - } - - // 1 directory is disconnected in the tree. dir1 was totally deleted so - // the repair tool will not see it. - Assertions.assertEquals(1, pendingDeleteDirCounts.size()); - Assertions.assertEquals(numWrites, pendingDeleteDirCounts.get("dir2")); - - // Check that disconnected files were put in deleting tables. - Map pendingDeleteFileCounts = new HashMap<>(); - - Table deletedFileTable = - leader.getMetadataManager().getDeletedTable(); - try (TableIterator> iterator = - deletedFileTable.iterator()) { - while (iterator.hasNext()) { - Table.KeyValue entry = iterator.next(); - String key = entry.getKey(); - RepeatedOmKeyInfo value = entry.getValue(); - - String[] keyParts = key.split("/"); - String fileName = keyParts[keyParts.length - 1]; - - LOG.info("In deletedTable, extracting file name {} from DB " + - "key {}", fileName, key); - - for (OmKeyInfo fileInfo: value.getOmKeyInfoList()) { - // Check that the correct file info was added. - Assertions.assertTrue(fileInfo.getKeyName().contains(fileName)); + }, 1000, 120000); + GenericTestUtils.waitFor(() -> { + try { + return leader.getMetadataManager().getDeletedTable().isEmpty(); + } catch (Exception e) { + LOG.error("DB failure!", e); + fail("DB failure!"); + return false; + } + }, 1000, 120000); + } - int count = pendingDeleteFileCounts.getOrDefault(fileName, 0); - pendingDeleteFileCounts.put(fileName, count + 1); - } + private void assertFileAndDirTablesEmpty() throws Exception { + OzoneManager leader = cluster.getOMLeader(); + GenericTestUtils.waitFor(() -> { + try { + return leader.getMetadataManager().getDirectoryTable().isEmpty(); + } catch (Exception e) { + LOG.error("DB failure!", e); + fail("DB failure!"); + return false; } - } + }, 1000, 120000); + GenericTestUtils.waitFor(() -> { + try { + return leader.getMetadataManager().getFileTable().isEmpty(); + } catch (Exception e) { + LOG.error("DB failure!", e); + fail("DB failure!"); + return false; + } + }, 1000, 120000); + } - // 3 files are disconnected in the tree. - // TODO: dir2 ended up in here with count = 1. file3 also had count=1 - // Likely that the dir2/file3 entry got split in two. - Assertions.assertEquals(3, pendingDeleteFileCounts.size()); - Assertions.assertEquals(numWrites, pendingDeleteFileCounts.get("file1")); - Assertions.assertEquals(numWrites, pendingDeleteFileCounts.get("file2")); - Assertions.assertEquals(numWrites, pendingDeleteFileCounts.get("file3")); + private DBStore getOmDB() { + return cluster.getOMLeader().getMetadataManager().getStore(); } - private void assertDeleteTablesEmpty() throws IOException { - OzoneManager leader = cluster.getOMLeader(); - Assertions.assertTrue(leader.getMetadataManager().getDeletedDirTable().isEmpty()); - Assertions.assertTrue(leader.getMetadataManager().getDeletedTable().isEmpty()); + private String getOmDBLocation() { + return cluster.getOMLeader().getMetadataManager().getStore().getDbLocation().toString(); + } + + private static BucketLayout getFSOBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; } private void runDeletes() throws Exception { @@ -623,17 +673,15 @@ private void runDeletes() throws Exception { } } - private void assertFileAndDirTablesEmpty() throws Exception { - OzoneManager leader = cluster.getOMLeader(); - Assertions.assertTrue(leader.getMetadataManager().getDirectoryTable().isEmpty()); - Assertions.assertTrue(leader.getMetadataManager().getFileTable().isEmpty()); - } - - private DBStore getOmDB() { - return cluster.getOMLeader().getMetadataManager().getStore(); - } - - private String getOmDBLocation() { - return cluster.getOMLeader().getMetadataManager().getStore().getDbLocation().toString(); + private int countTableEntries(Table table) throws Exception { + int count = 0; + try (TableIterator> iterator = table.iterator()) { + while (iterator.hasNext()) { + iterator.next(); + count++; + } + } + System.out.println("Total number of entries: " + count); + return count; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java index d866bd42786d..9a3418038a8d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairCLI.java @@ -47,12 +47,16 @@ public class FSORepairCLI implements Callable, SubcommandWithParent { @CommandLine.Option(names = {"--dry-run"}, defaultValue = "true", - description = "This tool will run in dry-run mode by default to log unreachable files or directories. " + - "Set the value to 'false' to move unreachable files and directories to the deleted tables.") + description = "Run in dry-run mode to log information about unreachable files or directories.") private boolean dryRun; + @CommandLine.Option(names = {"--repair"}, + defaultValue = "false", + description = "Run in repair mode to move unreachable files and directories to deleted tables.") + private boolean repair; + @CommandLine.Option(names = {"--volume"}, - description = "Filter by volume name") + description = "Filter by volume name. Add '/' before the volume name.") private String volume; @CommandLine.Option(names = {"--bucket"}, @@ -66,9 +70,15 @@ public class FSORepairCLI implements Callable, SubcommandWithParent { @Override public Void call() throws Exception { + if (repair) { + dryRun = false; //Disable dry-run if repair is passed. + System.out.println("FSO Repair Tool is running in repair mode"); + } else { + System.out.println("FSO Repair Tool is running in debug mode"); + } try { FSORepairTool - repairTool = new FSORepairTool(dbPath, dryRun, volume, bucket); + repairTool = new FSORepairTool(dbPath, dryRun, repair, volume, bucket); repairTool.run(); } catch (Exception ex) { throw new IllegalArgumentException("FSO repair failed: " + ex.getMessage()); @@ -86,4 +96,3 @@ public Class getParentType() { return OzoneRepair.class; } } - diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java index a0b917f255f3..08a6b4a50a23 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.ratis.util.Preconditions; @@ -95,6 +96,7 @@ public class FSORepairTool { private final Table fileTable; private final Table deletedDirectoryTable; private final Table deletedTable; + private final Table snapshotInfoTable; private final String volumeFilter; private final String bucketFilter; // The temporary DB is used to track which items have been seen. @@ -113,10 +115,10 @@ public class FSORepairTool { private long unreachableBytes; private long unreachableFiles; private long unreachableDirs; - private boolean dryRun; + private final boolean dryRun; - public FSORepairTool(String dbPath, boolean dryRun, String volume, String bucket) throws IOException { - this(getStoreFromPath(dbPath), dbPath, dryRun, volume, bucket); + public FSORepairTool(String dbPath, boolean dryRun, boolean repair, String volume, String bucket) throws IOException { + this(getStoreFromPath(dbPath), dbPath, dryRun, repair, volume, bucket); } /** @@ -124,7 +126,7 @@ public FSORepairTool(String dbPath, boolean dryRun, String volume, String bucket * class for testing. */ @VisibleForTesting - public FSORepairTool(DBStore dbStore, String dbPath, boolean isDryRun, String volume, String bucket) + public FSORepairTool(DBStore dbStore, String dbPath, boolean isDryRun, boolean isRepair, String volume, String bucket) throws IOException { dryRun = isDryRun; // Counters to track as we walk the tree. @@ -159,6 +161,10 @@ public FSORepairTool(DBStore dbStore, String dbPath, boolean isDryRun, String vo OmMetadataManagerImpl.DELETED_TABLE, String.class, RepeatedOmKeyInfo.class); + snapshotInfoTable = store.getTable( + OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE, + String.class, + SnapshotInfo.class); } protected static DBStore getStoreFromPath(String dbPath) throws IOException { @@ -173,7 +179,6 @@ protected static DBStore getStoreFromPath(String dbPath) throws IOException { } public org.apache.hadoop.ozone.repair.om.FSORepairTool.Report run() throws IOException { - System.out.println("Disclaimer: This tool currently does not support snapshots."); if (bucketFilter != null && volumeFilter == null) { System.out.println("--bucket flag cannot be used without specifying --volume."); @@ -217,6 +222,17 @@ public org.apache.hadoop.ozone.repair.om.FSORepairTool.Report run() throws IOExc continue; } + // Check for snapshots in the specified bucket + if (checkIfSnapshotExistsForBucket(volumeFilter, bucketFilter)) { + if (dryRun) { + System.out.println("Snapshot detected in bucket '" + bucketFilter + "'"); + } else { + System.out.println("Snapshot exists in bucket '" + bucketFilter + "'. " + + "Repair is not allowed if snapshots exist."); + return null; + } + } + processBucket(volumeEntry.getValue(), bucketInfo); } else { @@ -253,7 +269,37 @@ public org.apache.hadoop.ozone.repair.om.FSORepairTool.Report run() throws IOExc return buildReportAndLog(); } + private boolean checkIfSnapshotExistsForBucket(String volumeName, String bucketName) throws IOException { + if (snapshotInfoTable == null) { + return false; + } + + try (TableIterator> iterator = + snapshotInfoTable.iterator()) { + while (iterator.hasNext()) { + SnapshotInfo snapshotInfo = iterator.next().getValue(); + String snapshotPath = (volumeName + "/" + bucketName).replaceFirst("^/", ""); + if (snapshotInfo.getSnapshotPath().equals(snapshotPath)) { + return true; + } + } + } + return false; + } + private void processBucket(OmVolumeArgs volume, OmBucketInfo bucketInfo) throws IOException { + System.out.println("Processing bucket: " + volume.getVolume() + "/" + bucketInfo.getBucketName()); + if (checkIfSnapshotExistsForBucket(volume.getVolume(), bucketInfo.getBucketName())) { + if (dryRun) { + System.out.println( + "Snapshot detected in bucket '" + volume.getVolume() + "/" + bucketInfo.getBucketName() + "'. "); + } else { + System.out.println( + "Skipping repair for bucket '" + volume.getVolume() + "/" + bucketInfo.getBucketName() + "' " + + "due to snapshot presence."); + return; + } + } dropReachableTableIfExists(); createReachableTable(); markReachableObjectsInBucket(volume, bucketInfo); @@ -277,7 +323,6 @@ private Report buildReportAndLog() { private void markReachableObjectsInBucket(OmVolumeArgs volume, OmBucketInfo bucket) throws IOException { - System.out.println("Processing bucket: " + volume.getVolume() + "/" + bucket.getBucketName()); // Only put directories in the stack. // Directory keys should have the form /volumeID/bucketID/parentID/name. Stack dirKeyStack = new Stack<>(); @@ -335,6 +380,8 @@ private void handleUnreachableObjects(OmVolumeArgs volume, OmBucketInfo bucket) if (dryRun) { System.out.println("Marking unreachable directory " + dirKey + " for deletion."); + } else { + System.out.println("Deleting unreachable directory " + dirKey); OmDirectoryInfo dirInfo = dirEntry.getValue(); markDirectoryForDeletion(volume.getVolume(), bucket.getBucketName(), dirKey, dirInfo); @@ -363,6 +410,8 @@ private void handleUnreachableObjects(OmVolumeArgs volume, OmBucketInfo bucket) if (dryRun) { System.out.println("Marking unreachable file " + fileKey + " for deletion." + fileKey); + } else { + System.out.println("Deleting unreachable file " + fileKey); markFileForDeletion(fileKey, fileInfo); } } else { @@ -384,7 +433,7 @@ protected void markFileForDeletion(String fileKey, OmKeyInfo fileInfo) throws IO RepeatedOmKeyInfo updatedRepeatedOmKeyInfo = OmUtils.prepareKeyForDelete( fileInfo, fileInfo.getUpdateID(), true); // NOTE: The FSO code seems to write the open key entry with the whole - // path, using the object's names instead of their ID. This would onyl + // path, using the object's names instead of their ID. This would only // be possible when the file is deleted explicitly, and not part of a // directory delete. It is also not possible here if the file's parent // is gone. The name of the key does not matter so just use IDs. @@ -517,12 +566,19 @@ private void openReachableDB() throws IOException { private RocksDatabase buildReachableRocksDB(File reachableDBFile) throws IOException { DBProfile profile = new OzoneConfiguration().getEnum(HDDS_DB_PROFILE, HDDS_DEFAULT_DB_PROFILE); Set tableConfigs = new HashSet<>(); - tableConfigs.add(new TableConfig("default", profile.getColumnFamilyOptions())); - return RocksDatabase.open(reachableDBFile, - profile.getDBOptions(), - new ManagedWriteOptions(), - tableConfigs, false); + try { + tableConfigs.add(new TableConfig("default", profile.getColumnFamilyOptions())); + + return RocksDatabase.open(reachableDBFile, + profile.getDBOptions(), + new ManagedWriteOptions(), + tableConfigs, false); + } finally { + for (TableConfig config : tableConfigs) { + config.close(); + } + } } private void closeReachableDB() {