Skip to content

Commit 40bf994

Browse files
author
slfan1989
committed
YARN-11212. [Federation] Add getNodeToLabels REST APIs for Router.
1 parent 5c84cb8 commit 40bf994

File tree

6 files changed

+117
-20
lines changed

6 files changed

+117
-20
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public class NodeLabelsInfo {
3333

3434
@XmlElement(name = "nodeLabelInfo")
3535
private ArrayList<NodeLabelInfo> nodeLabelsInfo =
36-
new ArrayList<NodeLabelInfo>();
36+
new ArrayList<>();
3737

3838
public NodeLabelsInfo() {
3939
// JAXB needs this
@@ -44,25 +44,32 @@ public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
4444
}
4545

4646
public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
47-
this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
47+
this.nodeLabelsInfo = new ArrayList<>();
4848
for (NodeLabel label : nodeLabels) {
4949
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
5050
}
5151
}
5252

5353
public NodeLabelsInfo(Set<String> nodeLabelsName) {
54-
this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
54+
this.nodeLabelsInfo = new ArrayList<>();
5555
for (String labelName : nodeLabelsName) {
5656
this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
5757
}
5858
}
5959

60+
public NodeLabelsInfo(HashSet<NodeLabel> nodeLabels) {
61+
this.nodeLabelsInfo = new ArrayList<>();
62+
for (NodeLabel label : nodeLabels) {
63+
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
64+
}
65+
}
66+
6067
public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
6168
return nodeLabelsInfo;
6269
}
6370

6471
public Set<NodeLabel> getNodeLabels() {
65-
Set<NodeLabel> nodeLabels = new HashSet<NodeLabel>();
72+
Set<NodeLabel> nodeLabels = new HashSet<>();
6673
for (NodeLabelInfo label : nodeLabelsInfo) {
6774
nodeLabels.add(NodeLabel.newInstance(label.getName(),
6875
label.getExclusivity()));
@@ -71,7 +78,7 @@ public Set<NodeLabel> getNodeLabels() {
7178
}
7279

7380
public List<String> getNodeLabelsName() {
74-
ArrayList<String> nodeLabelsName = new ArrayList<String>();
81+
ArrayList<String> nodeLabelsName = new ArrayList<>();
7582
for (NodeLabelInfo label : nodeLabelsInfo) {
7683
nodeLabelsName.add(label.getName());
7784
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,8 @@ public NodeToLabelsInfo() {
3838
public HashMap<String, NodeLabelsInfo> getNodeToLabels() {
3939
return nodeToLabels;
4040
}
41+
42+
public void setNodeToLabels(HashMap<String, NodeLabelsInfo> nodeToLabels) {
43+
this.nodeToLabels = nodeToLabels;
44+
}
4145
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1172,7 +1172,28 @@ public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
11721172
@Override
11731173
public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr)
11741174
throws IOException {
1175-
throw new NotImplementedException("Code is not implemented");
1175+
NodeToLabelsInfo nodeToLabelsInfo = new NodeToLabelsInfo();
1176+
Map<SubClusterId, SubClusterInfo> subClustersActive;
1177+
try {
1178+
subClustersActive = getActiveSubclusters();
1179+
} catch (NotFoundException e) {
1180+
LOG.error("Get all active sub cluster(s) error.", e);
1181+
return nodeToLabelsInfo;
1182+
}
1183+
1184+
try {
1185+
final HttpServletRequest hsrCopy = clone(hsr);
1186+
Class[] argsClasses = new Class[]{HttpServletRequest.class};
1187+
Object[] args = new Object[]{hsrCopy};
1188+
ClientMethod remoteMethod = new ClientMethod("getNodeToLabels", argsClasses, args);
1189+
Map<SubClusterInfo, NodeToLabelsInfo> nodeToLabelsInfoMap =
1190+
invokeConcurrent(subClustersActive.values(), remoteMethod, NodeToLabelsInfo.class);
1191+
nodeToLabelsInfo = RouterWebServiceUtil.mergeNodeToLabels(nodeToLabelsInfoMap.values());
1192+
} catch (Exception e) {
1193+
LOG.error("Failed to return GetNodeToLabels.", e);
1194+
}
1195+
1196+
return nodeToLabelsInfo;
11761197
}
11771198

11781199
@Override

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.util.List;
3131
import java.util.Map;
3232
import java.util.Map.Entry;
33+
import java.util.Collection;
34+
import java.util.HashSet;
3335
import java.util.concurrent.TimeUnit;
3436

3537
import javax.servlet.http.HttpServletRequest;
@@ -43,13 +45,16 @@
4345
import org.apache.hadoop.net.NetUtils;
4446
import org.apache.hadoop.security.UserGroupInformation;
4547
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
48+
import org.apache.hadoop.yarn.api.records.NodeLabel;
4649
import org.apache.hadoop.yarn.conf.YarnConfiguration;
4750
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
4851
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
4952
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
5053
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
5154
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
5255
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
56+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
57+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
5358
import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
5459
import org.apache.hadoop.yarn.webapp.BadRequestException;
5560
import org.apache.hadoop.yarn.webapp.ForbiddenException;
@@ -293,8 +298,8 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
293298
boolean returnPartialResult) {
294299
AppsInfo allApps = new AppsInfo();
295300

296-
Map<String, AppInfo> federationAM = new HashMap<String, AppInfo>();
297-
Map<String, AppInfo> federationUAMSum = new HashMap<String, AppInfo>();
301+
Map<String, AppInfo> federationAM = new HashMap<>();
302+
Map<String, AppInfo> federationUAMSum = new HashMap<>();
298303
for (AppInfo a : appsInfo) {
299304
// Check if this AppInfo is an AM
300305
if (a.getAMHostHttpAddress() != null) {
@@ -332,7 +337,7 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
332337
}
333338
}
334339

335-
allApps.addAll(new ArrayList<AppInfo>(federationAM.values()));
340+
allApps.addAll(new ArrayList<>(federationAM.values()));
336341
return allApps;
337342
}
338343

@@ -419,7 +424,7 @@ public static NodesInfo deleteDuplicateNodesInfo(ArrayList<NodeInfo> nodes) {
419424
nodesMap.put(node.getNodeId(), node);
420425
}
421426
}
422-
nodesInfo.addAll(new ArrayList<NodeInfo>(nodesMap.values()));
427+
nodesInfo.addAll(new ArrayList<>(nodesMap.values()));
423428
return nodesInfo;
424429
}
425430

@@ -509,4 +514,27 @@ protected static <T> String getMediaTypeFromHttpServletRequest(
509514
return header;
510515
}
511516

517+
public static NodeToLabelsInfo mergeNodeToLabels(Collection<NodeToLabelsInfo> nodeToLabelsInfos) {
518+
NodeToLabelsInfo result = new NodeToLabelsInfo();
519+
HashMap<String, NodeLabelsInfo> nodeToLabels = new HashMap<>();
520+
521+
nodeToLabelsInfos.stream().forEach(nodeToLabelsInfo ->{
522+
for (Map.Entry<String, NodeLabelsInfo> item : nodeToLabelsInfo.getNodeToLabels().entrySet()) {
523+
if (nodeToLabels.containsKey(item.getKey())) {
524+
NodeLabelsInfo a = nodeToLabels.get(item.getKey());
525+
NodeLabelsInfo b = item.getValue();
526+
527+
HashSet<NodeLabel> hashSet = new HashSet<>();
528+
hashSet.addAll(a.getNodeLabels());
529+
hashSet.addAll(b.getNodeLabels());
530+
nodeToLabels.put(item.getKey(), new NodeLabelsInfo(hashSet));
531+
} else {
532+
nodeToLabels.put(item.getKey(), item.getValue());
533+
}
534+
}
535+
});
536+
537+
result.setNodeToLabels(nodeToLabels);
538+
return result;
539+
}
512540
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.net.ConnectException;
2323
import java.util.HashSet;
2424
import java.util.Set;
25+
import java.util.HashMap;
2526
import java.util.concurrent.atomic.AtomicInteger;
2627

2728
import javax.servlet.http.HttpServletRequest;
@@ -42,16 +43,7 @@
4243
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
4344
import org.apache.hadoop.yarn.exceptions.YarnException;
4445
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
45-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
46-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
47-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
48-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
49-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
50-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
51-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
52-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
53-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
54-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
46+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
5547
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
5648
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
5749
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -279,4 +271,28 @@ public ContainersInfo getContainers(HttpServletRequest req, HttpServletResponse
279271

280272
return containers;
281273
}
274+
275+
@Override
276+
public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException {
277+
if (!isRunning) {
278+
throw new RuntimeException("RM is stopped");
279+
}
280+
String cpuLabel = "CPU";
281+
HashSet<String> cpuLabels = new HashSet<>();
282+
cpuLabels.add(cpuLabel);
283+
284+
String gpuLabel = "GPU";
285+
HashSet<String> gpuLabels = new HashSet<>();
286+
gpuLabels.add(gpuLabel);
287+
288+
NodeLabelsInfo cpuNode = new NodeLabelsInfo(cpuLabels);
289+
NodeLabelsInfo gpuNode = new NodeLabelsInfo(gpuLabels);
290+
291+
NodeToLabelsInfo info = new NodeToLabelsInfo();
292+
HashMap<String, NodeLabelsInfo> nodeLabels = new HashMap<>();
293+
nodeLabels.put("node1", cpuNode);
294+
nodeLabels.put("node2", gpuNode);
295+
info.setNodeToLabels(nodeLabels);
296+
return info;
297+
}
282298
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.io.IOException;
2222
import java.util.ArrayList;
2323
import java.util.List;
24+
import java.util.HashMap;
2425

2526
import javax.ws.rs.core.Response;
2627

@@ -49,6 +50,8 @@
4950
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
5051
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
5152
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
53+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
54+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
5255
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
5356
import org.apache.hadoop.yarn.util.MonotonicClock;
5457
import org.junit.Assert;
@@ -605,4 +608,22 @@ public void testGetContainersWrongFormat() {
605608

606609
Assert.assertTrue(response.getContainers().isEmpty());
607610
}
611+
612+
@Test
613+
public void testGetNodeToLabels() throws IOException {
614+
NodeToLabelsInfo info = interceptor.getNodeToLabels(null);
615+
HashMap<String, NodeLabelsInfo> map = info.getNodeToLabels();
616+
Assert.assertNotNull(map);
617+
Assert.assertEquals(2, map.size());
618+
619+
NodeLabelsInfo node1Value = map.getOrDefault("node1", null);
620+
Assert.assertNotNull(node1Value);
621+
Assert.assertEquals(1, node1Value.getNodeLabelsName().size());
622+
Assert.assertEquals("CPU", node1Value.getNodeLabelsName().get(0));
623+
624+
NodeLabelsInfo node2Value = map.getOrDefault("node2", null);
625+
Assert.assertNotNull(node2Value);
626+
Assert.assertEquals(1, node2Value.getNodeLabelsName().size());
627+
Assert.assertEquals("GPU", node2Value.getNodeLabelsName().get(0));
628+
}
608629
}

0 commit comments

Comments
 (0)