YARN-11579. Fix 'Physical Mem Used' and 'Physical VCores Used' are not displaying data. (#6123) Contributed by Shilun Fan.

Reviewed-by: Inigo Goiri <inigoiri@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>

Author: slfan1989

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java

@@ -54,7 +54,9 @@ public class ClusterMetricsInfo {
   private int containersPending;
 
   private long totalMB;
+  private long utilizedMB;
   private long totalVirtualCores;
+  private long utilizedVirtualCores;
   private int utilizedMBPercent;
   private int utilizedVirtualCoresPercent;
   private int rmSchedulerBusyPercent;
@@ -167,6 +169,7 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
         .getContainerAssignedPerSecond();
     this.rmEventQueueSize = clusterMetrics.getRmEventQueueSize();
     this.schedulerEventQueueSize = clusterMetrics.getSchedulerEventQueueSize();
+    this.utilizedVirtualCores = clusterMetrics.getUtilizedVirtualCores();
   }
 
   public int getAppsSubmitted() {
@@ -432,4 +435,20 @@ public int getRmEventQueueSize() {
   public int getSchedulerEventQueueSize() {
     return schedulerEventQueueSize;
   }
+
+  public long getUtilizedVirtualCores() {
+    return utilizedVirtualCores;
+  }
+
+  public void setUtilizedVirtualCores(long utilizedVirtualCores) {
+    this.utilizedVirtualCores = utilizedVirtualCores;
+  }
+
+  public long getUtilizedMB() {
+    return utilizedMB;
+  }
+
+  public void setUtilizedMB(long utilizedMB) {
+    this.utilizedMB = utilizedMB;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java

@@ -481,7 +481,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
       Exception {
     assertEquals("incorrect number of elements", 1, json.length());
     JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
-    assertEquals("incorrect number of elements", 35, clusterinfo.length());
+    assertEquals("incorrect number of elements", 37, clusterinfo.length());
     verifyClusterMetrics(
         clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
         clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java

@@ -506,10 +506,14 @@ public static void mergeMetrics(ClusterMetricsInfo metrics,
 
     metrics.setTotalMB(metrics.getTotalMB()
         + metricsResponse.getTotalMB());
+    metrics.setUtilizedMB(metrics.getUtilizedMB()
+        + metricsResponse.getUtilizedMB());
     metrics.setTotalVirtualCores(metrics.getTotalVirtualCores()
         + metricsResponse.getTotalVirtualCores());
     metrics.setTotalNodes(metrics.getTotalNodes()
         + metricsResponse.getTotalNodes());
+    metrics.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores()
+        + metricsResponse.getUtilizedVirtualCores());
     metrics.setLostNodes(metrics.getLostNodes()
         + metricsResponse.getLostNodes());
     metrics.setUnhealthyNodes(metrics.getUnhealthyNodes()
@@ -524,6 +528,14 @@ public static void mergeMetrics(ClusterMetricsInfo metrics,
         + metricsResponse.getActiveNodes());
     metrics.setShutdownNodes(metrics.getShutdownNodes()
         + metricsResponse.getShutdownNodes());
+
+    int utilizedVirtualCoresPercent = metrics.getTotalVirtualCores() <= 0 ? 0 :
+        (int) (metrics.getUtilizedVirtualCores() * 100 / metrics.getTotalVirtualCores());
+    metrics.setUtilizedVirtualCoresPercent(utilizedVirtualCoresPercent);
+
+    int utilizedMBPercent = metrics.getTotalMB() <= 0 ? 0 :
+        (int) (metrics.getUtilizedMB() * 100 / metrics.getTotalMB());
+    metrics.setUtilizedMBPercent(utilizedMBPercent);
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java

@@ -489,10 +489,16 @@ public void testMergeMetrics() {
     Assert.assertEquals(
         metricsResponse.getTotalMB() + metricsClone.getTotalMB(),
         metrics.getTotalMB());
+    Assert.assertEquals(
+        metricsResponse.getUtilizedMB() + metricsClone.getUtilizedMB(),
+        metrics.getUtilizedMB());
     Assert.assertEquals(
         metricsResponse.getTotalVirtualCores()
             + metricsClone.getTotalVirtualCores(),
         metrics.getTotalVirtualCores());
+    Assert.assertEquals(
+        metricsResponse.getUtilizedVirtualCores() + metricsClone.getUtilizedVirtualCores(),
+        metrics.getUtilizedVirtualCores());
     Assert.assertEquals(
         metricsResponse.getTotalNodes() + metricsClone.getTotalNodes(),
         metrics.getTotalNodes());
@@ -544,7 +550,9 @@ private ClusterMetricsInfo createClusterMetricsClone(
     metricsClone.setContainersPending(metrics.getPendingContainers());
 
     metricsClone.setTotalMB(metrics.getTotalMB());
+    metricsClone.setUtilizedMB(metrics.getUtilizedMB());
     metricsClone.setTotalVirtualCores(metrics.getTotalVirtualCores());
+    metricsClone.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores());
     metricsClone.setTotalNodes(metrics.getTotalNodes());
     metricsClone.setLostNodes(metrics.getLostNodes());
     metricsClone.setUnhealthyNodes(metrics.getUnhealthyNodes());
@@ -580,7 +588,9 @@ private void setUpClusterMetrics(ClusterMetricsInfo metrics, long seed) {
     metrics.setContainersPending(rand.nextInt(1000));
 
     metrics.setTotalMB(rand.nextInt(1000));
+    metrics.setUtilizedMB(metrics.getTotalMB() - rand.nextInt(100));
     metrics.setTotalVirtualCores(rand.nextInt(1000));
+    metrics.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores() - rand.nextInt(100));
     metrics.setTotalNodes(rand.nextInt(1000));
     metrics.setLostNodes(rand.nextInt(1000));
     metrics.setUnhealthyNodes(rand.nextInt(1000));