Adding the table vs store metric by consolidating

apache · May 14, 2020 · 29e5ac8 · 29e5ac8
1 parent 7560131
commit 29e5ac8
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 38 deletions.
diff --git a/...oop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/...oop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
@@ -61,7 +61,11 @@
 import static org.apache.hadoop.hbase.regionserver.MetricsRegionServerSource.SPLIT_SUCCESS_DESC;
 import static org.apache.hadoop.hbase.regionserver.MetricsRegionServerSource.SPLIT_SUCCESS_KEY;
 
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.metrics.Interns;
 import org.apache.hadoop.metrics2.MetricHistogram;
@@ -87,6 +91,8 @@ public class MetricsTableSourceImpl implements MetricsTableSource {
   private final MetricsTableAggregateSourceImpl agg;
   private final DynamicMetricsRegistry registry;
   private final String tableNamePrefix;
+  private final String tableNamePrefixPart1;
+  private final String tableNamePrefixPart2;
   private final TableName tableName;
   private final int hashCode;
 
@@ -127,8 +133,11 @@ public MetricsTableSourceImpl(String tblName,
 
     this.tableWrapperAgg = tblWrapperAgg;
     this.registry = agg.getMetricsRegistry();
-    this.tableNamePrefix = "Namespace_" + this.tableName.getNamespaceAsString() +
-        "_table_" + this.tableName.getQualifierAsString() + "_metric_";
+    this.tableNamePrefixPart1 = "Namespace_" + this.tableName.getNamespaceAsString() +
+            "_table_" + this.tableName.getQualifierAsString();
+    this.tableNamePrefixPart2 = "_metric_";
+    this.tableNamePrefix = tableNamePrefixPart1 +
+        tableNamePrefixPart2;
     this.hashCode = this.tableName.hashCode();
   }
 
@@ -311,12 +320,25 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) {
         mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
             MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC),
             tableWrapperAgg.getNumReferenceFiles(tableName.getNameAsString()));
-        mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionSource.GET_REQUEST_ON_MEMSTORE,
-          MetricsRegionSource.GET_REQUEST_ON_MEMSTORE_DESC),
-          tableWrapperAgg.getMemstoreReadRequestCount(tableName.getNameAsString()));
-        mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionSource.GET_REQUEST_ON_FILE,
-          MetricsRegionSource.GET_REQUEST_ON_FILE_DESC),
-          tableWrapperAgg.getFileRequestCount(tableName.getNameAsString()));
+        addGauge(mrb, tableWrapperAgg.getMemstoreReadRequestCount(tableName.getNameAsString()),
+          MetricsRegionSource.GET_REQUEST_ON_MEMSTORE,
+          MetricsRegionSource.GET_REQUEST_ON_MEMSTORE_DESC);
+        addGauge(mrb, tableWrapperAgg.getFileRequestCount(tableName.getNameAsString()),
+          MetricsRegionSource.GET_REQUEST_ON_FILE, MetricsRegionSource.GET_REQUEST_ON_FILE_DESC);
+      }
+    }
+  }
+
+  private void addGauge(MetricsRecordBuilder mrb, Map<String, Long> metricMap, String metricName,
+      String metricDesc) {
+    if (metricMap != null) {
+      Iterator<Entry<String, Long>> iterator = metricMap.entrySet().iterator();
+      while (iterator.hasNext()) {
+        Entry<String, Long> entry = iterator.next();
+        // append 'store' and its name to the metric
+        mrb.addGauge(Interns.info(this.tableNamePrefixPart1 + "_store_" + entry.getKey()
+            + this.tableNamePrefixPart2 + metricName,
+          metricDesc), entry.getValue());
       }
     }
   }

diff --git a/...mpat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java b/...mpat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.Map;
+
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -110,11 +112,10 @@ public interface MetricsTableWrapperAggregate {
   /**
    * @return number of get requests on memstore for this table
    */
-  long getMemstoreReadRequestCount(String table);
+  Map<String, Long> getMemstoreReadRequestCount(String table);
 
   /**
    * @return number of get requests from file for this table
    */
-  long getFileRequestCount(String table);
-
+  Map<String, Long> getFileRequestCount(String table);
 }
diff --git a/...op-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java b/...op-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate {
 
   private String tableName;
@@ -111,14 +114,16 @@ public long getCpRequestsCount(String table) {
   }
 
   @Override
-  public long getMemstoreReadRequestCount(String table) {
-    // TODO Auto-generated method stub
-    return 3;
+  public Map<String, Long> getMemstoreReadRequestCount(String table) {
+    Map<String, Long> map = new HashMap<String, Long>();
+    map.put("info", 3l);
+    return map;
   }
 
   @Override
-  public long getFileRequestCount(String table) {
-    // TODO Auto-generated method stub
-    return 3;
+  public Map<String, Long> getFileRequestCount(String table) {
+    Map<String, Long> map = new HashMap<String, Long>();
+    map.put("info", 3l);
+    return map;
   }
 }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsStoreWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsStoreWrapperImpl.java
@@ -54,9 +54,9 @@ public class MetricsStoreWrapperImpl implements MetricsStoreWrapper, Closeable {
 
   public MetricsStoreWrapperImpl(HStore store) {
     this.store = store;
-    ScheduledExecutorService executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
-    this.storeMetricUpdateTask =
-        executor.scheduleWithFixedDelay(new HStoreMetricsWrapperRunnable(),
+    ScheduledExecutorService executor =
+        CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
+    this.storeMetricUpdateTask = executor.scheduleWithFixedDelay(new HStoreMetricsWrapperRunnable(),
       PERIOD, PERIOD, TimeUnit.SECONDS);
   }
 

diff --git a/.../src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java b/.../src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java
@@ -61,7 +61,6 @@ public class TableMetricsWrapperRunnable implements Runnable {
     @Override
     public void run() {
       Map<TableName, MetricsTableValues> localMetricsTableMap = new HashMap<>();
-
       for (Region r : regionServer.getOnlineRegionsLocalContext()) {
         TableName tbl = r.getTableDescriptor().getTableName();
         MetricsTableValues mt = localMetricsTableMap.get(tbl);
@@ -70,21 +69,31 @@ public void run() {
           localMetricsTableMap.put(tbl, mt);
         }
         if (r.getStores() != null) {
+          long memstoreReadCount = 0l;
+          long fileReadCount = 0l;
+          String familyName = null;
           for (Store store : r.getStores()) {
+            familyName = store.getColumnFamilyName();
+
             mt.storeFileCount += store.getStorefilesCount();
-            mt.memstoreSize += (store.getMemStoreSize().getDataSize() +
-              store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize());
+            mt.memstoreSize += (store.getMemStoreSize().getDataSize()
+                + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize());
             mt.storeFileSize += store.getStorefilesSize();
             mt.referenceFileCount += store.getNumReferenceFiles();
 
-            mt.maxStoreFileAge = Math.max(mt.maxStoreFileAge, store.getMaxStoreFileAge().getAsLong());
-            mt.minStoreFileAge = Math.min(mt.minStoreFileAge, store.getMinStoreFileAge().getAsLong());
-            mt.totalStoreFileAge = (long)store.getAvgStoreFileAge().getAsDouble() *
-                store.getStorefilesCount();
+            mt.maxStoreFileAge =
+                Math.max(mt.maxStoreFileAge, store.getMaxStoreFileAge().getAsLong());
+            mt.minStoreFileAge =
+                Math.min(mt.minStoreFileAge, store.getMinStoreFileAge().getAsLong());
+            mt.totalStoreFileAge =
+                (long) store.getAvgStoreFileAge().getAsDouble() * store.getStorefilesCount();
             mt.storeCount += 1;
-            mt.memstoreGetCount += store.getGetRequestsCountFromMemstore();
-            mt.fileGetCount += store.getGetRequestsCountFromFile();
+            memstoreReadCount += store.getGetRequestsCountFromMemstore();
+            fileReadCount += store.getGetRequestsCountFromFile();
+            mt.storeMemstoreGetCount.putIfAbsent(familyName, memstoreReadCount);
+            mt.storeFileGetCount.putIfAbsent(familyName, fileReadCount);
           }
+
           mt.regionCount += 1;
 
           mt.readRequestCount += r.getReadRequestsCount();
@@ -113,7 +122,7 @@ public void run() {
         if (metricsTableMap.get(table) != null) {
           metricsTableMap.remove(table);
         }
-      }
+      } 
     }
   }
 
@@ -128,22 +137,22 @@ public long getReadRequestCount(String table) {
   }
 
   @Override
-  public long getMemstoreReadRequestCount(String table) {
+  public Map<String, Long> getMemstoreReadRequestCount(String table) {
     MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
     if (metricsTable == null) {
-      return 0;
+      return null;
     } else {
-      return metricsTable.memstoreGetCount;
+      return metricsTable.storeMemstoreGetCount;
     }
   }
 
   @Override
-  public long getFileRequestCount(String table) {
+  public Map<String, Long> getFileRequestCount(String table) {
     MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
     if (metricsTable == null) {
-      return 0;
+      return null;
     } else {
-      return metricsTable.fileGetCount;
+      return metricsTable.storeFileGetCount;
     }
   }
 
@@ -318,8 +327,8 @@ private static class MetricsTableValues {
     long totalStoreFileAge;
     long referenceFileCount;
     long cpRequestCount;
-    long memstoreGetCount;
-    long fileGetCount;
+    Map<String, Long> storeMemstoreGetCount = new HashMap<>();
+    Map<String, Long> storeFileGetCount = new HashMap<>();
   }
 
 }