HBASE-25754 StripeCompactionPolicy should support compacting cold reg…

…ions (#3152) Signed-off-by: Duo Zhang <zhangduo@apache.org>
apache · Apr 23, 2021 · e6cb7f2 · e6cb7f2
1 parent 50bdced
commit e6cb7f2
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 1 deletion.
diff --git a/...rc/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java b/...rc/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
@@ -162,7 +162,7 @@ public boolean needsCompactions(StripeInformationProvider si, List<HStoreFile> f
     return filesCompacting.isEmpty()
         && (StoreUtils.hasReferences(si.getStorefiles())
           || (si.getLevel0Files().size() >= this.config.getLevel0MinFiles())
-          || needsSingleStripeCompaction(si));
+          || needsSingleStripeCompaction(si) || hasExpiredStripes(si));
   }
 
   @Override
@@ -338,6 +338,33 @@ private StripeCompactionRequest selectExpiredMergeCompaction(
     return result;
   }
 
+  private boolean isStripeExpired(ImmutableList<HStoreFile> storeFiles) {
+    long cfTtl = this.storeConfigInfo.getStoreFileTtl();
+    if (cfTtl == Long.MAX_VALUE) {
+      return false; // minversion might be set, cannot delete old files
+    }
+    long timestampCutoff = EnvironmentEdgeManager.currentTime() - cfTtl;
+    for (HStoreFile storeFile : storeFiles) {
+      // Check store file is not empty and has not expired
+      if (storeFile.getReader().getMaxTimestamp() >= timestampCutoff
+        && storeFile.getReader().getEntries() != 0) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  protected boolean hasExpiredStripes(StripeInformationProvider si) {
+    // Find if exists a stripe where all files have expired, if any.
+    ArrayList<ImmutableList<HStoreFile>> stripes = si.getStripes();
+    for (ImmutableList<HStoreFile> stripe : stripes) {
+      if (isStripeExpired(stripe)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private static long getTotalKvCount(final Collection<HStoreFile> candidates) {
     long totalSize = 0;
     for (HStoreFile storeFile : candidates) {

diff --git a/...est/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/...est/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
@@ -35,6 +35,7 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,6 +52,7 @@
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.HStore;
@@ -83,6 +85,7 @@
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 import org.mockito.ArgumentMatcher;
+
 import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 
@@ -283,6 +286,35 @@ public void testNothingToCompactFromL0() throws Exception {
     verifyNoCompaction(policy, si);
   }
 
+  @Test
+  public void testCheckExpiredStripeCompaction() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 5);
+    conf.setInt(StripeStoreConfig.MIN_FILES_KEY, 4);
+
+    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
+    long now = defaultTtl + 2;
+    edge.setValue(now);
+    EnvironmentEdgeManager.injectEdge(edge);
+    HStoreFile expiredFile = createFile(10), notExpiredFile = createFile(10);
+    when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
+    when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
+    List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
+    List<HStoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile);
+
+    StripeCompactionPolicy policy =
+      createPolicy(conf, defaultSplitSize, defaultSplitCount, defaultInitialCount, true);
+    // Merge expired if there are eligible stripes.
+    StripeCompactionPolicy.StripeInformationProvider si =
+      createStripesWithFiles(mixed, mixed, mixed);
+    assertFalse(policy.needsCompactions(si, al()));
+
+    si = createStripesWithFiles(mixed, mixed, mixed, expired);
+    assertFalse(policy.needsSingleStripeCompaction(si));
+    assertTrue(policy.hasExpiredStripes(si));
+    assertTrue(policy.needsCompactions(si, al()));
+  }
+
   @Test
   public void testSplitOffStripe() throws Exception {
     Configuration conf = HBaseConfiguration.create();
@@ -771,6 +803,7 @@ private static HStoreFile createFile(long size) throws Exception {
       anyBoolean())).thenReturn(mock(StoreFileScanner.class));
     when(sf.getReader()).thenReturn(r);
     when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty());
+    when(r.getMaxTimestamp()).thenReturn(TimeRange.INITIAL_MAX_TIMESTAMP);
     return sf;
   }