diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 391c7d9b1914..0eac3d617b00 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -36,6 +37,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate; +import org.apache.hadoop.hbase.master.region.MasterRegionFactory; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.wal.WAL; @@ -123,27 +125,8 @@ public Iterable getDeletableFiles(Iterable files) { return Collections.emptyList(); } for (FileStatus file : files) { - String fn = file.getPath().getName(); - if (fn.startsWith(WALProcedureStore.LOG_PREFIX)) { + if (canDeleteFile(addressToLastBackupMap, file.getPath())) { filteredFiles.add(file); - continue; - } - - try { - Address walServerAddress = - Address.fromString(BackupUtils.parseHostNameFromLogFile(file.getPath())); - long walTimestamp = WAL.getTimestamp(file.getPath().getName()); - - if ( - !addressToLastBackupMap.containsKey(walServerAddress) - || addressToLastBackupMap.get(walServerAddress) >= walTimestamp - ) { - filteredFiles.add(file); - } - } catch (Exception ex) { - LOG.warn( - "Error occurred while filtering file: {} with error: {}. Ignoring cleanup of this log", - file.getPath(), ex.getMessage()); } } @@ -176,4 +159,39 @@ public void stop(String why) { public boolean isStopped() { return this.stopped; } + + protected static boolean canDeleteFile(Map addressToLastBackupMap, Path path) { + if (isHMasterWAL(path)) { + return true; + } + + try { + String hostname = BackupUtils.parseHostNameFromLogFile(path); + if (hostname == null) { + LOG.warn( + "Cannot parse hostname from RegionServer WAL file: {}. Ignoring cleanup of this log", + path); + return false; + } + Address walServerAddress = Address.fromString(hostname); + long walTimestamp = WAL.getTimestamp(path.getName()); + + if ( + !addressToLastBackupMap.containsKey(walServerAddress) + || addressToLastBackupMap.get(walServerAddress) >= walTimestamp + ) { + return true; + } + } catch (Exception ex) { + LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", path, ex); + return false; + } + return false; + } + + private static boolean isHMasterWAL(Path path) { + String fn = path.getName(); + return fn.startsWith(WALProcedureStore.LOG_PREFIX) + || fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX); + } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java index 2b0f9c0cba5f..e372c6ad1533 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java @@ -20,10 +20,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupType; @@ -132,4 +134,21 @@ public void testBackupLogCleaner() throws Exception { conn.close(); } } + + @Test + public void testCleansUpHMasterWal() { + Path path = new Path("/hbase/MasterData/WALs/hmaster,60000,1718808578163"); + assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), path)); + } + + @Test + public void testCleansUpArchivedHMasterWal() { + Path normalPath = + new Path("/hbase/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); + assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), normalPath)); + + Path masterPath = new Path( + "/hbase/MasterData/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); + assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), masterPath)); + } }