HBASE-23213 : Reopen regions with very high Store Ref Counts(backport…

hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java

@@ -165,6 +165,13 @@ public int getDeadServers() {
     return deadServers != null ? deadServers.size() : 0;
   }
 
+  /**
+   * @return map of the names of region servers on the live list with associated ServerLoad
+   */
+  public Map<ServerName, ServerLoad> getLiveServersLoad() {
+    return Collections.unmodifiableMap(liveServers);
+  }
+
   /**
    * @return the average cluster load
    */

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java

@@ -194,6 +194,14 @@ public int getStoreRefCount() {
     return regionLoadPB.getStoreRefCount();
   }
 
+  /**
+   * @return the max reference count for any store file among all stores files
+   *   of this region
+   */
+  public int getMaxStoreFileRefCount() {
+    return regionLoadPB.getMaxStoreFileRefCount();
+  }
+
   /**
    * @see java.lang.Object#toString()
    */
@@ -204,6 +212,7 @@ public String toString() {
     sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
         this.getStorefiles());
     sb = Strings.appendKeyValue(sb, "storeRefCount", this.getStoreRefCount());
+    sb = Strings.appendKeyValue(sb, "maxStoreFileRefCount", this.getMaxStoreFileRefCount());
     sb = Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
       this.getStoreUncompressedSizeMB());
     sb = Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -1341,6 +1341,13 @@ public static enum Modify {
   // User defined Default TTL config key
   public static final String DEFAULT_SNAPSHOT_TTL_CONFIG_KEY = "hbase.master.snapshot.ttl";
 
+  // Regions Recovery based on high storeFileRefCount threshold value
+  public static final String STORE_FILE_REF_COUNT_THRESHOLD =
+    "hbase.regions.recovery.store.file.ref.count";
+
+  // default -1 indicates there is no threshold on high storeRefCount
+  public static final int DEFAULT_STORE_FILE_REF_COUNT_THRESHOLD = -1;
+
   /**
    * Configurations for master executor services.
    */

hbase-common/src/main/resources/hbase-default.xml

@@ -1643,4 +1643,33 @@ possible configurations would overwhelm and obscure the important.
       automatically deleted until it is manually deleted
     </description>
   </property>
+  <property>
+    <name>hbase.master.regions.recovery.check.interval</name>
+    <value>1200000</value>
+    <description>
+      Regions Recovery Chore interval in milliseconds.
+      This chore keeps running at this interval to
+      find all regions with configurable max store file ref count
+      and reopens them.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regions.recovery.store.file.ref.count</name>
+    <value>-1</value>
+    <description>
+      Very large ref count on a file indicates
+      that it is a ref leak on that object. Such files
+      can not be removed even after it is invalidated
+      via compaction. Only way to recover in such
+      scenario is to reopen the region which can
+      release all resources, like the refcount, leases, etc.
+      This config represents Store files Ref Count threshold
+      value considered for reopening regions.
+      Any region with store files ref count > this value
+      would be eligible for reopening by master.
+      Default value -1 indicates this feature is turned off.
+      Only positive integer value should be provided to enable
+      this feature.
+    </description>
+  </property>
 </configuration>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -231,6 +231,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String STOREFILE_COUNT_DESC = "Number of Store Files";
   String STORE_REF_COUNT = "storeRefCount";
   String STORE_REF_COUNT_DESC = "Store reference count";
+  String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount";
   String MEMSTORE_SIZE = "memStoreSize";
   String MEMSTORE_SIZE_DESC = "Size of the memstore";
   String STOREFILE_SIZE = "storeFileSize";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java

@@ -147,4 +147,10 @@ public interface MetricsRegionWrapper {
    * @return the number of references active on the store
    */
   long getStoreRefCount();
+
+  /**
+   * @return the max reference count for any store file among all stores files
+   *   of this region
+   */
+  int getMaxStoreFileRefCount();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

@@ -217,6 +217,10 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) {
               regionNamePrefix + MetricsRegionServerSource.STORE_REF_COUNT,
               MetricsRegionServerSource.STORE_REF_COUNT),
           this.regionWrapper.getStoreRefCount());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT,
+        MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT),
+        this.regionWrapper.getMaxStoreFileRefCount());
       mrb.addGauge(Interns.info(
               regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
               MetricsRegionServerSource.MEMSTORE_SIZE_DESC),

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java

@@ -96,6 +96,11 @@ public long getStoreRefCount() {
       return 0;
     }
 
+    @Override
+    public int getMaxStoreFileRefCount() {
+      return 0;
+    }
+
     @Override
     public long getMemstoreSize() {
       return 0;