HBASE-23349 : Expose max refCount among all compacted store files and…

… reopen region for threshold value

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java

@@ -160,4 +160,10 @@ default String getNameAsString() {
    *   of this region
    */
   int getMaxStoreFileRefCount();
+
+  /**
+   * @return the max reference count for any store file among all compacted stores files
+   *   of this region
+   */
+  int getMaxCompactedStoreFileRefCount();
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java

@@ -67,6 +67,7 @@ public static RegionMetrics toRegionMetrics(ClusterStatusProtos.RegionLoad regio
         .setStoreFileCount(regionLoadPB.getStorefiles())
         .setStoreRefCount(regionLoadPB.getStoreRefCount())
         .setMaxStoreFileRefCount(regionLoadPB.getMaxStoreFileRefCount())
+        .setMaxCompactedStoreFileRefCount(regionLoadPB.getMaxCompactedStoreFileRefCount())
         .setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE))
         .setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
           .collect(Collectors.toMap(
@@ -115,6 +116,7 @@ public static ClusterStatusProtos.RegionLoad toRegionLoad(RegionMetrics regionMe
         .setStorefiles(regionMetrics.getStoreFileCount())
         .setStoreRefCount(regionMetrics.getStoreRefCount())
         .setMaxStoreFileRefCount(regionMetrics.getMaxStoreFileRefCount())
+        .setMaxCompactedStoreFileRefCount(regionMetrics.getMaxCompactedStoreFileRefCount())
         .setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
         .addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
         .setStoreUncompressedSizeMB(
@@ -131,6 +133,7 @@ public static RegionMetricsBuilder newBuilder(byte[] name) {
   private int storeFileCount;
   private int storeRefCount;
   private int maxStoreFileRefCount;
+  private int maxCompactedStoreFileRefCount;
   private long compactingCellCount;
   private long compactedCellCount;
   private Size storeFileSize = Size.ZERO;
@@ -168,6 +171,10 @@ public RegionMetricsBuilder setMaxStoreFileRefCount(int value) {
     this.maxStoreFileRefCount = value;
     return this;
   }
+  public RegionMetricsBuilder setMaxCompactedStoreFileRefCount(int value) {
+    this.maxCompactedStoreFileRefCount = value;
+    return this;
+  }
   public RegionMetricsBuilder setCompactingCellCount(long value) {
     this.compactingCellCount = value;
     return this;
@@ -243,6 +250,7 @@ public RegionMetrics build() {
         storeFileCount,
         storeRefCount,
         maxStoreFileRefCount,
+        maxCompactedStoreFileRefCount,
         compactingCellCount,
         compactedCellCount,
         storeFileSize,
@@ -268,6 +276,7 @@ private static class RegionMetricsImpl implements RegionMetrics {
     private final int storeFileCount;
     private final int storeRefCount;
     private final int maxStoreFileRefCount;
+    private final int maxCompactedStoreFileRefCount;
     private final long compactingCellCount;
     private final long compactedCellCount;
     private final Size storeFileSize;
@@ -290,6 +299,7 @@ private static class RegionMetricsImpl implements RegionMetrics {
         int storeFileCount,
         int storeRefCount,
         int maxStoreFileRefCount,
+        int maxCompactedStoreFileRefCount,
         final long compactingCellCount,
         long compactedCellCount,
         Size storeFileSize,
@@ -312,6 +322,7 @@ private static class RegionMetricsImpl implements RegionMetrics {
       this.storeFileCount = storeFileCount;
       this.storeRefCount = storeRefCount;
       this.maxStoreFileRefCount = maxStoreFileRefCount;
+      this.maxCompactedStoreFileRefCount = maxCompactedStoreFileRefCount;
       this.compactingCellCount = compactingCellCount;
       this.compactedCellCount = compactedCellCount;
       this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@@ -356,6 +367,11 @@ public int getMaxStoreFileRefCount() {
       return maxStoreFileRefCount;
     }
 
+    @Override
+    public int getMaxCompactedStoreFileRefCount() {
+      return maxCompactedStoreFileRefCount;
+    }
+
     @Override
     public Size getStoreFileSize() {
       return storeFileSize;
@@ -451,6 +467,8 @@ public String toString() {
         this.getStoreRefCount());
       Strings.appendKeyValue(sb, "maxStoreFileRefCount",
         this.getMaxStoreFileRefCount());
+      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
+        this.getMaxCompactedStoreFileRefCount());
       Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
           this.getUncompressedStoreFileSize());
       Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java

@@ -345,6 +345,7 @@ public String toString() {
       int storeFileCount = 0;
       int storeRefCount = 0;
       int maxStoreFileRefCount = 0;
+      int maxCompactedStoreFileRefCount = 0;
       long uncompressedStoreFileSizeMB = 0;
       long storeFileSizeMB = 0;
       long memStoreSizeMB = 0;
@@ -363,6 +364,9 @@ public String toString() {
         storeRefCount += r.getStoreRefCount();
         int currentMaxStoreFileRefCount = r.getMaxStoreFileRefCount();
         maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
+        int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
+        maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+          currentMaxCompactedStoreFileRefCount);
         uncompressedStoreFileSizeMB += r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
         storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE);
         memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE);
@@ -386,6 +390,8 @@ public String toString() {
       Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
       Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
       Strings.appendKeyValue(sb, "maxStoreFileRefCount", maxStoreFileRefCount);
+      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
+        maxCompactedStoreFileRefCount);
       Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
       Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
       if (uncompressedStoreFileSizeMB != 0) {

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -234,6 +234,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String STORE_REF_COUNT = "storeRefCount";
   String STORE_REF_COUNT_DESC = "Store reference count";
   String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount";
+  String MAX_COMPACTED_STORE_FILE_REF_COUNT = "maxCompactedStoreFileRefCount";
   String MEMSTORE_SIZE = "memStoreSize";
   String MEMSTORE_SIZE_DESC = "Size of the memstore";
   String STOREFILE_SIZE = "storeFileSize";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java

@@ -170,4 +170,10 @@ public interface MetricsRegionWrapper {
    *   all store files that belong to this region
    */
   long getMaxStoreFileRefCount();
+
+  /**
+   * @return the max number of references active on any store file among
+   *   all compacted store files that belong to this region
+   */
+  long getMaxCompactedStoreFileRefCount();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

@@ -221,6 +221,11 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) {
         regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT,
         MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT),
         this.regionWrapper.getMaxStoreFileRefCount());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT,
+        MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT),
+        this.regionWrapper.getMaxCompactedStoreFileRefCount()
+      );
       mrb.addGauge(Interns.info(
               regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
               MetricsRegionServerSource.MEMSTORE_SIZE_DESC),

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java

@@ -104,6 +104,11 @@ public long getMaxStoreFileRefCount() {
       return 0;
     }
 
+    @Override
+    public long getMaxCompactedStoreFileRefCount() {
+      return 0;
+    }
+
     @Override
     public long getMemStoreSize() {
       return 0;

hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto

@@ -155,6 +155,12 @@ message RegionLoad {
    *  that belong to given region
    */
   optional int32 max_store_file_ref_count = 22 [default = 0];
+
+  /**
+   *  The max number of references active on single store file among all compacted store files
+   *  that belong to given region
+   */
+  optional int32 max_compacted_store_file_ref_count = 23 [default = 0];
 }
 
 /* Server-level protobufs */

hbase-protocol/src/main/protobuf/ClusterStatus.proto

@@ -151,6 +151,12 @@ message RegionLoad {
    *  that belong to given region
    */
   optional int32 max_store_file_ref_count = 22 [default = 0];
+
+  /**
+   *  The max number of references active on single store file among all compacted store files
+   *  that belong to given region
+   */
+  optional int32 max_compacted_store_file_ref_count = 23 [default = 0];
 }
 
 /* Server-level protobufs */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java

@@ -130,12 +130,12 @@ private Map<TableName, List<byte[]>> getTableToRegionsByRefCount(
     for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
       Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
       for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
-        // For each region, each store file can have different ref counts
-        // We need to find maximum of all such ref counts and if that max count
-        // is beyond a threshold value, we should reopen the region.
-        // Here, we take max ref count of all store files and not the cumulative
-        // count of all store files
-        final int maxStoreFileRefCount = regionMetrics.getMaxStoreFileRefCount();
+        // For each region, each compacted store file can have different ref counts
+        // We need to find maximum of all such ref counts and if that max count of compacted
+        // store files is beyond a threshold value, we should reopen the region.
+        // Here, we take max ref count of all compacted store files and not the cumulative
+        // count of all compacted store files
+        final int maxStoreFileRefCount = regionMetrics.getMaxCompactedStoreFileRefCount();
 
         if (maxStoreFileRefCount > storeFileRefCountThreshold) {
           final byte[] regionName = regionMetrics.getRegionName();

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1630,6 +1630,7 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr,
     int storefiles = 0;
     int storeRefCount = 0;
     int maxStoreFileRefCount = 0;
+    int maxCompactedStoreFileRefCount = 0;
     int storeUncompressedSizeMB = 0;
     int storefileSizeMB = 0;
     int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024);
@@ -1647,6 +1648,9 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr,
       storeRefCount += currentStoreRefCount;
       int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
       maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
+      int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
+      maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+        currentMaxCompactedStoreFileRefCount);
       storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024);
       storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
       //TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB?
@@ -1676,6 +1680,7 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr,
       .setStorefiles(storefiles)
       .setStoreRefCount(storeRefCount)
       .setMaxStoreFileRefCount(maxStoreFileRefCount)
+      .setMaxCompactedStoreFileRefCount(maxCompactedStoreFileRefCount)
       .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
       .setStorefileSizeMB(storefileSizeMB)
       .setMemStoreSizeMB(memstoreSizeMB)

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java

@@ -2798,4 +2798,20 @@ public int getMaxStoreFileRefCount() {
     return maxStoreFileRefCount.isPresent() ? maxStoreFileRefCount.getAsInt() : 0;
   }
 
+  /**
+   * @return get maximum ref count of storeFile among all compacted HStore Files
+   *   for the HStore
+   */
+  public int getMaxCompactedStoreFileRefCount() {
+    OptionalInt maxCompactedStoreFileRefCount = this.storeEngine.getStoreFileManager()
+      .getCompactedfiles()
+      .stream()
+      .filter(sf -> sf.getReader() != null)
+      .filter(HStoreFile::isHFile)
+      .mapToInt(HStoreFile::getRefCount)
+      .max();
+    return maxCompactedStoreFileRefCount.isPresent()
+      ? maxCompactedStoreFileRefCount.getAsInt() : 0;
+  }
+
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java

@@ -50,6 +50,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
   private long numStoreFiles;
   private long storeRefCount;
   private long maxStoreFileRefCount;
+  private long maxCompactedStoreFileRefCount;
   private long memstoreSize;
   private long storeFileSize;
   private long maxStoreFileAge;
@@ -131,6 +132,11 @@ public long getMaxStoreFileRefCount() {
     return maxStoreFileRefCount;
   }
 
+  @Override
+  public long getMaxCompactedStoreFileRefCount() {
+    return maxCompactedStoreFileRefCount;
+  }
+
   @Override
   public long getReadRequestCount() {
     return this.region.getReadRequestsCount();
@@ -240,6 +246,7 @@ public void run() {
       long tempNumStoreFiles = 0;
       int tempStoreRefCount = 0;
       int tempMaxStoreFileRefCount = 0;
+      int tempMaxCompactedStoreFileRefCount = 0;
       long tempMemstoreSize = 0;
       long tempStoreFileSize = 0;
       long tempMaxStoreFileAge = 0;
@@ -257,6 +264,9 @@ public void run() {
           int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
           tempMaxStoreFileRefCount = Math.max(tempMaxStoreFileRefCount,
             currentMaxStoreFileRefCount);
+          int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
+          tempMaxCompactedStoreFileRefCount = Math.max(tempMaxCompactedStoreFileRefCount,
+            currentMaxCompactedStoreFileRefCount);
           tempMemstoreSize += store.getMemStoreSize().getDataSize();
           tempStoreFileSize += store.getStorefilesSize();
           OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -285,6 +295,7 @@ public void run() {
       numStoreFiles = tempNumStoreFiles;
       storeRefCount = tempStoreRefCount;
       maxStoreFileRefCount = tempMaxStoreFileRefCount;
+      maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
       memstoreSize = tempMemstoreSize;
       storeFileSize = tempStoreFileSize;
       maxStoreFileAge = tempMaxStoreFileAge;

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java

@@ -375,7 +375,7 @@ public long getLastReportTimestamp() {
     return serverMetrics;
   }
 
-  private static RegionMetrics getRegionMetrics(byte[] regionName, int storeRefCount) {
+  private static RegionMetrics getRegionMetrics(byte[] regionName, int compactedStoreRefCount) {
     RegionMetrics regionMetrics = new RegionMetrics() {
 
       @Override
@@ -480,12 +480,17 @@ public long getLastMajorCompactionTimestamp() {
 
       @Override
       public int getStoreRefCount() {
-        return storeRefCount;
+        return compactedStoreRefCount;
       }
 
       @Override
       public int getMaxStoreFileRefCount() {
-        return storeRefCount;
+        return 0;
+      }
+
+      @Override
+      public int getMaxCompactedStoreFileRefCount() {
+        return compactedStoreRefCount;
       }
 
     };
@@ -610,4 +615,4 @@ public boolean isStopped() {
 
   }
 
-}
+}

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java

@@ -70,6 +70,11 @@ public long getMaxStoreFileRefCount() {
     return 0;
   }
 
+  @Override
+  public long getMaxCompactedStoreFileRefCount() {
+    return 0;
+  }
+
   @Override
   public long getMemStoreSize() {
     return 103;