HBASE-28742 Fixes NPE for CompactionTool when mslab enabled

apache · Jul 23, 2024 · d7b397e · d7b397e
1 parent b0240ce
commit d7b397e
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 0 deletions.
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -211,6 +211,8 @@ public void setup(Context context) {
       } catch (IOException e) {
         throw new RuntimeException("Could not get the input FileSystem", e);
       }
+      // Disable the MemStoreLAB as MemStore is not used by flow during compaction
+      conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
     }
 
     @Override
@@ -369,6 +371,8 @@ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
    */
   private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
     final boolean compactOnce, final boolean major) throws IOException {
+    // Disable the MemStoreLAB as MemStore is not used by flow during compaction
+    getConf().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
     CompactionWorker worker = new CompactionWorker(fs, getConf());
     for (Path path : toCompactDirs) {
       worker.compact(path, compactOnce, major);

diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionTool.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionTool.java
@@ -19,6 +19,8 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.Arrays;
+import java.util.Collection;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -38,8 +40,11 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 @Category({ MediumTests.class, RegionServerTests.class })
+@RunWith(Parameterized.class)
 public class TestCompactionTool {
 
   @ClassRule
@@ -53,8 +58,17 @@ public class TestCompactionTool {
   private Path rootDir;
   private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
 
+  @Parameterized.Parameter
+  public boolean mslab;
+
+  @Parameterized.Parameters(name = "mslab{0}")
+  public static synchronized Collection<Boolean> data() {
+    return Arrays.asList(true, false);
+  }
+
   @Before
   public void setUp() throws Exception {
+    testUtil.getConfiguration().setBoolean(MemStoreLAB.USEMSLAB_KEY, mslab);
     this.testUtil.startMiniCluster();
     testUtil.createTable(tableName, HBaseTestingUtil.fam1);
     rootDir = testUtil.getDefaultRootDirPath();
@@ -97,4 +111,33 @@ private void putAndFlush(int key) throws Exception {
     region.flush(true);
   }
 
+  @Test
+  public void testCompactedFilesArchivedMapRed() throws Exception {
+    for (int i = 0; i < 10; i++) {
+      this.putAndFlush(i);
+    }
+    HStore store = region.getStore(HBaseTestingUtil.fam1);
+    assertEquals(10, store.getStorefilesCount());
+    Path tableDir = CommonFSUtils.getTableDir(rootDir, region.getRegionInfo().getTable());
+    FileSystem fs = store.getFileSystem();
+    String storePath = tableDir + "/" + region.getRegionInfo().getEncodedName() + "/"
+      + Bytes.toString(HBaseTestingUtil.fam1);
+    FileStatus[] regionDirFiles = fs.listStatus(new Path(storePath));
+    assertEquals(10, regionDirFiles.length);
+    String defaultFS = testUtil.getMiniHBaseCluster().getConfiguration().get("fs.defaultFS");
+    testUtil.startMiniMapReduceCluster();
+    try {
+      Configuration config = HBaseConfiguration.create(testUtil.getConfiguration());
+      config.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
+      config.set("fs.defaultFS", defaultFS);
+      int result = ToolRunner.run(config, new CompactionTool(),
+        new String[] { "-compactOnce", "-mapred", "-major", storePath });
+      assertEquals(0, result);
+      regionDirFiles = fs.listStatus(new Path(storePath));
+      assertEquals(1, regionDirFiles.length);
+    } finally {
+      testUtil.shutdownMiniMapReduceCluster();
+    }
+  }
+
 }