HBASE-25847 More DEBUG and TRACE level logging in CatalogJanitor and …

…HbckChore
apache · May 5, 2021 · 1ec2d05 · 1ec2d05
1 parent 17193da
commit 1ec2d05
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 15 deletions.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java
@@ -223,15 +223,37 @@ private void loadRegionsFromInMemoryState() {
           .isTableState(regionInfo.getTable(), TableState.State.DISABLED)) {
         disabledTableRegions.add(regionInfo.getRegionNameAsString());
       }
-      if (regionInfo.isSplitParent()) {
+      if (regionState.isSplit()) {
         splitParentRegions.add(regionInfo.getRegionNameAsString());
       }
       HbckRegionInfo.MetaEntry metaEntry =
           new HbckRegionInfo.MetaEntry(regionInfo, regionState.getServerName(),
               regionState.getStamp());
       regionInfoMap.put(regionInfo.getEncodedName(), new HbckRegionInfo(metaEntry));
     }
-    LOG.info("Loaded {} regions from in-memory state of AssignmentManager", regionStates.size());
+    LOG.info("Loaded {} regions ({} disabled, {} split parents) from in-memory state",
+      regionStates.size(), disabledTableRegions.size(), splitParentRegions.size());
+    if (LOG.isDebugEnabled()) {
+      Map<RegionState.State,Integer> stateCountMap = new HashMap<>();
+      for (RegionState regionState : regionStates) {
+        stateCountMap.compute(regionState.getState(), (k, v) -> (v == null) ? 1 : v + 1);
+      }
+      StringBuffer sb = new StringBuffer();
+      sb.append("Regions by state: ");
+      stateCountMap.entrySet().forEach(e -> {
+        sb.append(e.getKey());
+        sb.append('=');
+        sb.append(e.getValue());
+        sb.append(' ');
+        });
+      LOG.debug(sb.toString());
+    }
+    if (LOG.isTraceEnabled()) {
+      for (RegionState regionState : regionStates) {
+        LOG.trace("{}: {}, serverName=", regionState.getRegion(), regionState.getState(),
+          regionState.getServerName());
+      }
+    }
   }
 
   private void loadRegionsFromRSReport() {

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
@@ -74,6 +74,8 @@
 @InterfaceAudience.Private
 public class CatalogJanitor extends ScheduledChore {
 
+  public static final int DEFAULT_HBASE_CATALOGJANITOR_INTERVAL = 300 * 1000;
+
   private static final Logger LOG = LoggerFactory.getLogger(CatalogJanitor.class.getName());
 
   private final AtomicBoolean alreadyRunning = new AtomicBoolean(false);
@@ -88,7 +90,8 @@ public class CatalogJanitor extends ScheduledChore {
 
   public CatalogJanitor(final MasterServices services) {
     super("CatalogJanitor-" + services.getServerName().toShortString(), services,
-      services.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
+      services.getConfiguration().getInt("hbase.catalogjanitor.interval",
+        DEFAULT_HBASE_CATALOGJANITOR_INTERVAL));
     this.services = services;
   }
 
@@ -165,13 +168,19 @@ public int scan() throws IOException {
       this.lastReport = scanForReport();
       if (!this.lastReport.isEmpty()) {
         LOG.warn(this.lastReport.toString());
+      } else {
+        LOG.debug(this.lastReport.toString());
       }
+
       updateAssignmentManagerMetrics();
 
       Map<RegionInfo, Result> mergedRegions = this.lastReport.mergedRegions;
       for (Map.Entry<RegionInfo, Result> e : mergedRegions.entrySet()) {
         if (this.services.isInMaintenanceMode()) {
           // Stop cleaning if the master is in maintenance mode
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("In maintenence mode, not cleaning");
+          }
           break;
         }
 
@@ -188,6 +197,9 @@ public int scan() throws IOException {
       for (Map.Entry<RegionInfo, Result> e : splitParents.entrySet()) {
         if (this.services.isInMaintenanceMode()) {
           // Stop cleaning if the master is in maintenance mode
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("In maintenence mode, not cleaning");
+          }
           break;
         }
 
@@ -235,6 +247,7 @@ public Report getLastReport() {
    */
   private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents)
     throws IOException {
+    LOG.debug("Cleaning merged region {}", mergedRegion);
     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
     Path rootdir = this.services.getMasterFileSystem().getRootDir();
     Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable());
@@ -252,14 +265,11 @@ private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo>
           .stream().map(r -> RegionInfo.getShortNameToLog(r)).collect(Collectors.joining(", ")),
         mergedRegion);
       ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
-      pe.submitProcedure(
-        new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents));
-      for (RegionInfo ri : parents) {
-        // The above scheduled GCMultipleMergedRegionsProcedure does the below.
-        // Do we need this?
-        this.services.getAssignmentManager().getRegionStates().deleteRegion(ri);
-        this.services.getServerManager().removeRegion(ri);
-      }
+      GCMultipleMergedRegionsProcedure mergeRegionProcedure =
+          new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents);
+      pe.submitProcedure(mergeRegionProcedure);
+      LOG.debug("Submitted procedure {} for merged region {}", mergeRegionProcedure,
+        mergedRegion);
       return true;
     }
     return false;
@@ -300,9 +310,11 @@ public int compare(RegionInfo left, RegionInfo right) {
 
   static boolean cleanParent(MasterServices services, RegionInfo parent, Result rowContent)
     throws IOException {
+    LOG.debug("Cleaning parent region {}", parent);
     // Check whether it is a merged region and if it is clean of references.
     if (CatalogFamilyFormat.hasMergeRegions(rowContent.rawCells())) {
       // Wait until clean of merge parent regions first
+      LOG.debug("Region {} has merge parents, cleaning them first", parent);
       return false;
     }
     // Run checks on each daughter split.
@@ -317,11 +329,19 @@ static boolean cleanParent(MasterServices services, RegionInfo parent, Result ro
       LOG.debug("Deleting region " + parent.getShortNameToLog() + " because daughters -- " +
         daughterA + ", " + daughterB + " -- no longer hold references");
       ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor();
-      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
-      // Remove from in-memory states
-      services.getAssignmentManager().getRegionStates().deleteRegion(parent);
-      services.getServerManager().removeRegion(parent);
+      GCRegionProcedure gcRegionProcedure = new GCRegionProcedure(pe.getEnvironment(), parent);
+      pe.submitProcedure(gcRegionProcedure);
+      LOG.debug("Submitted procedure {} for split parent {}", gcRegionProcedure, parent);
       return true;
+    } else {
+      if (!hasNoReferences(a)) {
+        LOG.debug("Deferring removal of region {} because daughter {} still has references",
+          parent, daughters.getFirst());
+      }
+      if (!hasNoReferences(b)) {
+        LOG.debug("Deferring removal of region {} because daughter {} still has references",
+          parent, daughters.getSecond());
+      }
     }
     return false;
   }