[YUNIKORN-2325] Add a chart to display multi-type resource utilization (

#160)

Closes: #160

Signed-off-by: Craig Condit <ccondit@apache.org>

json-routes.json

@@ -1,5 +1,6 @@
 {
   "/ws/v1/scheduler/node-utilization": "/node-utilization",
+  "/ws/v1/scheduler/node-utilizations": "/node-utilizations",
   "/ws/v1/*": "/$1",
   "/history/apps": "/appHistory",
   "/history/containers": "/containerHistory",

src/app/app.module.ts

@@ -61,6 +61,8 @@ import { StatusViewComponent } from '@app/components/status-view/status-view.com
 import { HealthchecksComponent } from '@app/components/healthchecks/healthchecks.component';
 import { AppNodeUtilizationComponent } from '@app/components/app-node-utilization/app-node-utilization.component';
 import { BarChartComponent } from '@app/components/bar-chart/bar-chart.component';
+import { AppNodeUtilizationsComponent } from '@app/components/app-node-utilizations/app-node-utilizations.component';
+import { VerticalBarChartComponent } from '@app/components/vertical-bar-chart/vertical-bar-chart.component';
 
 @NgModule({
   declarations: [
@@ -82,6 +84,8 @@ import { BarChartComponent } from '@app/components/bar-chart/bar-chart.component
     HealthchecksComponent,
     AppNodeUtilizationComponent,
     BarChartComponent,
+    AppNodeUtilizationsComponent,
+    VerticalBarChartComponent,
   ],
   imports: [
     BrowserModule,

src/app/components/app-node-utilizations/app-node-utilizations.component.html

@@ -0,0 +1,30 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+<mat-card appearance="outlined" class="box-card">
+  <mat-card-header>
+    <mat-card-title>Node Resource Utilization</mat-card-title>
+  </mat-card-header>
+  <mat-card-content>
+    <div class="status-wrapper flex-grid">
+      <div class="chart-wrapper flex-primary">
+        <app-vertical-bar-chart [bucketList]="bucketList" [barChartDataSets]="barChartDataSets" />
+      </div>
+    </div>
+  </mat-card-content>
+</mat-card>

src/app/components/app-node-utilizations/app-node-utilizations.component.scss

@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

src/app/components/app-node-utilizations/app-node-utilizations.component.spec.ts

@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { MatCardModule } from '@angular/material/card';
+import { AppNodeUtilizationsComponent } from '@app/components/app-node-utilizations/app-node-utilizations.component';
+import { VerticalBarChartComponent } from '@app/components/vertical-bar-chart/vertical-bar-chart.component';
+import { CHART_COLORS } from '@app/utils/constants';
+
+describe('AppNodeUtilizationsComponent', () => {
+  let component: AppNodeUtilizationsComponent;
+  let fixture: ComponentFixture<AppNodeUtilizationsComponent>;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({
+      imports: [HttpClientTestingModule, MatCardModule],
+      declarations: [AppNodeUtilizationsComponent, VerticalBarChartComponent]
+    });
+
+    fixture = TestBed.createComponent(AppNodeUtilizationsComponent);
+    component = fixture.componentInstance;
+  });
+
+  it('test AppNodeUtilizationsComponent.calculateAvgUtilization()', () => {
+    type TestCase = {
+      description: string;
+      nodeNumInBuckets: number[];
+      expected: number;
+    };
+    const testCases: TestCase[] = [
+      {
+        description: 'Test 2 nodes, 1 node in 0~10%, 1 node in 10~20%',
+        nodeNumInBuckets: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
+        expected: 0.1
+      },
+      {
+        description: 'Test 10 nodes, 1 node in each bucket',
+        nodeNumInBuckets: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+        expected: 0.5
+      },
+      {
+        description: 'Test zero node in buckets',
+        nodeNumInBuckets: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        expected: 0
+      },
+    ]
+
+    testCases.forEach((testCase: TestCase) => {
+      const result = component.calculateAvgUtilization(testCase.nodeNumInBuckets);
+      expect(result).toEqual(testCase.expected);
+    });
+  });
+
+  it('test AppNodeUtilizationsComponent.generateColorMapping()', () => {
+    const types = [
+      'type03', 'type01', 'type02', 'type04', 'type05',
+      'type06', 'type07', 'type08', 'type09', 'type10', 'type11'
+    ];
+    const colorMapping = component.generateColorMapping(types);
+
+    expect(colorMapping.size).toBe(11);
+    expect(colorMapping.get('type01')).toBe(CHART_COLORS[0]);
+    expect(colorMapping.get('type02')).toBe(CHART_COLORS[1]);
+    expect(colorMapping.get('type03')).toBe(CHART_COLORS[2]);
+    expect(colorMapping.get('type11')).toBe(CHART_COLORS[0]);
+  });
+
+  it('test AppNodeUtilizationsComponent.getBarDescription()', () => {
+    type TestCase = {
+      description: string;
+      nodeNames: string[];
+      expected: string;
+    };
+    const testCases: TestCase[] = [
+      {
+        description: 'Test single node',
+        nodeNames: [""],
+        expected: ""
+      },
+      {
+        description: 'Test unordered multi-nodes',
+        nodeNames: ["node02", "node01"],
+        expected: "node01\nnode02"
+      },
+      {
+        description: 'Test over than MAX_NODES_IN_DESCRIPTION nodes',
+        nodeNames: ["node01", "node02", "node03", "node04", "node05", "node06", "node07", "node08", "node09", "node10", "node11", "node12", "node13", "node14", "node15", "node16"],
+        expected: "node01\nnode02\nnode03\nnode04\nnode05\nnode06\nnode07\nnode08\nnode09\nnode10\nnode11\nnode12\nnode13\nnode14\nnode15\n...1 more"
+      },
+    ]
+
+    testCases.forEach((testCase: TestCase) => {
+      const result = component.getBarDescription(testCase.nodeNames);
+      expect(result).toEqual(testCase.expected);
+    });
+  });
+});

src/app/components/app-node-utilizations/app-node-utilizations.component.ts

@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core';
+import { BarChartDataSet } from '@app/models/chart-data.model';
+import { CHART_COLORS, DEFAULT_BAR_COLOR } from '@app/utils/constants';
+import { CommonUtil } from '@app/utils/common.util';
+import { NodeUtilization, NodeUtilizationsInfo } from '@app/models/node-utilization.model';
+import { SchedulerService } from '@app/services/scheduler/scheduler.service';
+
+
+@Component({
+  selector: 'app-node-utilizations',
+  templateUrl: './app-node-utilizations.component.html',
+  styleUrls: ['./app-node-utilizations.component.scss']
+})
+export class AppNodeUtilizationsComponent implements OnInit, OnChanges {
+  nodeUtilizations: NodeUtilization[] = [];
+
+  // input data for vertical bar chart, key is resource type
+  bucketList: string[] = [];                                          // one bucket list for all resource types, length should be exactly 10
+  barChartDataSets: BarChartDataSet[] = new Array<BarChartDataSet>(); // one dataset for each type
+
+  @Input() partitionSelected: string = "";
+
+  constructor(
+    private scheduler: SchedulerService
+  ) { }
+
+  ngOnInit() {
+    this.reloadBarChartData()
+  }
+
+  ngOnChanges(changes: SimpleChanges) {
+    if (
+      changes['partitionSelected']
+    ) {
+      this.reloadBarChartData()
+    }
+  }
+
+  reloadBarChartData() {
+    this.scheduler.fetchNodeUtilizationsInfo().subscribe((data) => {
+      let nodeUtilizationsInfo: NodeUtilizationsInfo[] = data
+      for (let i = 0; i < nodeUtilizationsInfo.length; i++) {
+        if (nodeUtilizationsInfo[i].partition === this.partitionSelected) {
+          let nodeUtilizations = nodeUtilizationsInfo[i].utilizations
+          this.fetchBarChartData(nodeUtilizations)
+          break;
+        }
+      }
+    });
+  }
+
+  fetchBarChartData(nodeUtilizations: NodeUtilization[]) {
+    let barChartDataSets = new Array<BarChartDataSet>();
+    if (nodeUtilizations.length === 0) {
+      // clean bar chart data
+      this.barChartDataSets = barChartDataSets;
+      return;
+    }
+
+    let colorMapping = this.generateColorMapping(
+      nodeUtilizations.map((nodeUtilization) => (nodeUtilization.type))
+    );
+
+    for (let i = 0; i < nodeUtilizations.length; i++) {
+      let type = nodeUtilizations[i].type;
+      let utilization = nodeUtilizations[i].utilization
+      let borderWidth = 1
+
+      if (i === 0) {
+        // get bucketList only from the first type of node utilization
+        // should always be 10 buckets. (ranging from 0% to 100%).
+        this.bucketList = utilization.map((item) => item.bucketName);
+      }
+      let bucketValues = utilization.map((item) => item.numOfNodes);
+      barChartDataSets.push(new BarChartDataSet(
+        type,
+        bucketValues,
+        this.calculateAvgUtilization(bucketValues),
+        colorMapping.get(type) ?? DEFAULT_BAR_COLOR,
+        borderWidth,
+        utilization.map((item) => this.getBarDescription(item.nodeNames))
+      ))
+    }
+
+    // sort by resource type first, then sort by avg utilization rate
+    barChartDataSets.sort((a, b) => CommonUtil.resourcesCompareFn(a.label, b.label));
+    barChartDataSets.sort((a, b) => b.avgUtilizationRate - a.avgUtilizationRate);
+    barChartDataSets = barChartDataSets.slice(0, 10); // only show top 10 resources
+
+    // refresh bar chart data
+    this.barChartDataSets = barChartDataSets;
+  }
+
+  calculateAvgUtilization(nodeNumInBuckets: number[]): number {
+    // Calculates the average utilization of nodes based on a distribution of node utilizations.
+    // Note: It not a precise average.
+    // value of nodeCounts[0] means node count of 0%~10%, take 5% as the utilization of node in bucket
+    // value of nodeCounts[1] means node count of 10%~20, take 15% as the utilization of node in bucket
+    // value of nodeCounts[9] means node count of 90%~100%, take 95% as the utilization of node in bucket
+    let totalNodes = 0;
+    let weightedSum = 0;
+    for (let i = 0; i < 10; i++) { //buckets have fixed length 10
+      if (nodeNumInBuckets[i] != undefined) {
+        totalNodes += nodeNumInBuckets[i];
+        weightedSum += nodeNumInBuckets[i] * (5 + 10 * i);
+      }
+    }
+    return totalNodes ? weightedSum / totalNodes / 100 : 0;
+  }
+
+  generateColorMapping(types: string[]): Map<string, string> {
+    // give each resource type a color based on its index after lexicographically sorting
+    types.sort();
+    let colorMapping = new Map<string, string>();
+    for (let i = 0; i < types.length; i++) {
+      colorMapping.set(types[i], CHART_COLORS[i % 10])
+    }
+    return colorMapping
+  }
+
+  getBarDescription(nodeNames: string[] | null): string {
+    let MAX_NODES_IN_DESCRIPTION = 15;
+    let description: string | undefined;
+    if (nodeNames && nodeNames.length > MAX_NODES_IN_DESCRIPTION) {
+      // only put MAX_NODES_IN_DESCRIPTION nodes in description, others will be replaced by '...N more'
+      description = nodeNames.slice(0, MAX_NODES_IN_DESCRIPTION).sort().join("\n") + "\n..." + (nodeNames.length - MAX_NODES_IN_DESCRIPTION) + " more";
+    } else {
+      description = nodeNames ? nodeNames.sort().join("\n") : undefined;
+    }
+    return description || ""
+  }
+}

src/app/components/dashboard/dashboard.component.html

@@ -95,7 +95,7 @@
   </div>
   <div class="flex-grid grid-row">
     <div class="half-col flex-primary">
-      <app-node-utilization [chartData]="nodeUtilizationChartData.chartDataItems" title="Nodes Resource Utilization" [subtitle]="('Domaint Resource:' + nodeUtilizationChartData.type)"/>
+      <app-node-utilization [chartData]="nodeUtilizationChartData.chartDataItems" title="Node Resource Utilization" [subtitle]="('Dominant Resource:' + nodeUtilizationChartData.type)"/>
     </div>
   </div>
 </div>

src/app/components/dashboard/dashboard.component.ts

@@ -109,7 +109,7 @@ export class DashboardComponent implements OnInit {
       this.appHistoryData = this.getAreaChartData(data);
     });
 
-    this.scheduler.fetchClusterNodeUtilization().subscribe((data) => {
+    this.scheduler.fetchNodeUtilization().subscribe((data) => {
       let nodeUtilization = new NodeUtilization(data.type, data.utilization);
       this.nodeUtilizationChartData = nodeUtilization.toNodeUtilizationChartData();
     });

src/app/components/nodes-view/nodes-view.component.html

@@ -197,4 +197,9 @@ <h3>Allocations</h3>
       ></mat-paginator>
     </div>
   </div>
+  <div class="flex-grid">
+    <div class="flex-primary">
+      <app-node-utilizations [partitionSelected]="partitionSelected" />
+    </div>
+  </div>
 </div>

src/app/components/nodes-view/nodes-view.component.spec.ts

@@ -21,9 +21,13 @@ import { NoopAnimationsModule } from '@angular/platform-browser/animations';
 import { NgxSpinnerService } from 'ngx-spinner';
 
 import { NodesViewComponent } from './nodes-view.component';
+import { AppNodeUtilizationsComponent } from '@app/components/app-node-utilizations/app-node-utilizations.component';
+import { VerticalBarChartComponent } from '@app/components/vertical-bar-chart/vertical-bar-chart.component';
+
 import { SchedulerService } from '@app/services/scheduler/scheduler.service';
 import { HAMMER_LOADER } from '@angular/platform-browser';
 import { MockSchedulerService, MockNgxSpinnerService } from '@app/testing/mocks';
+import { MatCardModule } from '@angular/material/card';
 import { MatTableModule } from '@angular/material/table';
 import { MatPaginatorModule } from '@angular/material/paginator';
 import { MatDividerModule } from '@angular/material/divider';
@@ -58,13 +62,12 @@ describe('NodesViewComponent', () => {
 
   beforeEach(() => {
     TestBed.configureTestingModule({
-      declarations: [NodesViewComponent],
-      imports: [MatFormFieldModule, MatInputModule, ReactiveFormsModule],
+      declarations: [NodesViewComponent, AppNodeUtilizationsComponent, VerticalBarChartComponent],
+      imports: [MatFormFieldModule, MatInputModule, ReactiveFormsModule, MatCardModule],
     })
     .compileComponents();
     fixture = TestBed.createComponent(NodesViewComponent);
     component = fixture.componentInstance;
-    fixture.detectChanges();
   });
 
   it('should create the component', () => {