Skip to content

Commit 16e6030

Browse files
tomicoolertomicooler
authored
YARN-10891. Extend QueueInfo with max-parallel-apps in CS. (#3314)
Co-authored-by: Tamas Domok <tdomok@cloudera.com>
1 parent 4c94831 commit 16e6030

File tree

15 files changed

+88
-16
lines changed

15 files changed

+88
-16
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ public static QueueInfo newInstance(String queueName,
6060
List<QueueInfo> childQueues, List<ApplicationReport> applications,
6161
QueueState queueState, Set<String> accessibleNodeLabels,
6262
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
63-
boolean preemptionDisabled, float weight) {
63+
boolean preemptionDisabled, float weight,
64+
int maxParallelApps) {
6465
QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
6566
queueInfo.setQueueName(queueName);
6667
queueInfo.setQueuePath(queuePath);
@@ -75,6 +76,7 @@ public static QueueInfo newInstance(String queueName,
7576
queueInfo.setQueueStatistics(queueStatistics);
7677
queueInfo.setPreemptionDisabled(preemptionDisabled);
7778
queueInfo.setWeight(weight);
79+
queueInfo.setMaxParallelApps(maxParallelApps);
7880
return queueInfo;
7981
}
8082

@@ -86,14 +88,14 @@ public static QueueInfo newInstance(String queueName,
8688
List<QueueInfo> childQueues, List<ApplicationReport> applications,
8789
QueueState queueState, Set<String> accessibleNodeLabels,
8890
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
89-
boolean preemptionDisabled, float weight,
91+
boolean preemptionDisabled, float weight, int maxParallelApps,
9092
Map<String, QueueConfigurations> queueConfigurations) {
9193
QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
9294
maximumCapacity, currentCapacity,
9395
childQueues, applications,
9496
queueState, accessibleNodeLabels,
9597
defaultNodeLabelExpression, queueStatistics,
96-
preemptionDisabled, weight);
98+
preemptionDisabled, weight, maxParallelApps);
9799
queueInfo.setQueueConfigurations(queueConfigurations);
98100
return queueInfo;
99101
}
@@ -106,15 +108,15 @@ public static QueueInfo newInstance(String queueName,
106108
List<QueueInfo> childQueues, List<ApplicationReport> applications,
107109
QueueState queueState, Set<String> accessibleNodeLabels,
108110
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
109-
boolean preemptionDisabled, float weight,
111+
boolean preemptionDisabled, float weight, int maxParallelApps,
110112
Map<String, QueueConfigurations> queueConfigurations,
111113
boolean intraQueuePreemptionDisabled) {
112114
QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
113115
maximumCapacity, currentCapacity,
114116
childQueues, applications,
115117
queueState, accessibleNodeLabels,
116118
defaultNodeLabelExpression, queueStatistics,
117-
preemptionDisabled, weight, queueConfigurations);
119+
preemptionDisabled, weight, maxParallelApps, queueConfigurations);
118120
queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled);
119121
return queueInfo;
120122
}
@@ -166,6 +168,18 @@ public static QueueInfo newInstance(String queueName,
166168
@Private
167169
@Unstable
168170
public abstract void setWeight(float weight);
171+
172+
/**
173+
* Get the <em>configured max parallel apps</em> of the queue.
174+
* @return <em>configured max parallel apps</em> of the queue
175+
*/
176+
@Public
177+
@Stable
178+
public abstract int getMaxParallelApps();
179+
180+
@Private
181+
@Unstable
182+
public abstract void setMaxParallelApps(int maxParallelApps);
169183

170184
/**
171185
* Get the <em>maximum capacity</em> of the queue.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ message QueueInfoProto {
617617
optional bool intraQueuePreemptionDisabled = 13;
618618
optional float weight = 14;
619619
optional string queuePath = 15;
620+
optional int32 maxParallelApps = 16;
620621
}
621622

622623
message QueueConfigurationsProto {

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) {
139139
writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
140140
writer.print("\tWeight : ");
141141
writer.println(df.format(queueInfo.getWeight()));
142+
writer.print("\tMaximum Parallel Apps : ");
143+
writer.println(queueInfo.getMaxParallelApps());
142144
writer.print("\tDefault Node Label expression : ");
143145
String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression();
144146
nodeLabelExpression =

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,7 @@ public List<NodeReport> createFakeNodeReports() {
670670
public QueueInfo createFakeQueueInfo() {
671671
return QueueInfo.newInstance("root", "root", 100f, 100f, 50f, null,
672672
createFakeAppReports(), QueueState.RUNNING, null,
673-
null, null, false, -1.0f,
673+
null, null, false, -1.0f, 10,
674674
null, false);
675675
}
676676

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1722,8 +1722,8 @@ public void testGetQueueInfo() throws Exception {
17221722
newInstance("queueA", "root.queueA",
17231723
0.4f, 0.8f, 0.5f,
17241724
null, null, QueueState.RUNNING, nodeLabels,
1725-
"GPU", null, false, -1.0f, null,
1726-
false);
1725+
"GPU", null, false, -1.0f, 10,
1726+
null, false);
17271727
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
17281728
int result = cli.run(new String[] { "-status", "queueA" });
17291729
assertEquals(0, result);
@@ -1738,6 +1738,7 @@ public void testGetQueueInfo() throws Exception {
17381738
pw.println("\tCurrent Capacity : " + "50.00%");
17391739
pw.println("\tMaximum Capacity : " + "80.00%");
17401740
pw.println("\tWeight : " + "-1.00");
1741+
pw.println("\tMaximum Parallel Apps : " + "10");
17411742
pw.println("\tDefault Node Label expression : " + "GPU");
17421743
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
17431744
pw.println("\tPreemption : " + "enabled");
@@ -1895,7 +1896,7 @@ public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
18951896
newInstance("queueA", "root.queueA",
18961897
0.4f, 0.8f, 0.5f,
18971898
null, null, QueueState.RUNNING, null, null, null,
1898-
true, -1.0f, null, true);
1899+
true, -1.0f, 10, null, true);
18991900
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
19001901
int result = cli.run(new String[] { "-status", "queueA" });
19011902
assertEquals(0, result);
@@ -1910,6 +1911,7 @@ public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
19101911
pw.println("\tCurrent Capacity : " + "50.00%");
19111912
pw.println("\tMaximum Capacity : " + "80.00%");
19121913
pw.println("\tWeight : " + "-1.00");
1914+
pw.println("\tMaximum Parallel Apps : " + "10");
19131915
pw.println("\tDefault Node Label expression : "
19141916
+ NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
19151917
pw.println("\tAccessible Node Labels : ");

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,18 @@ public void setWeight(float weight) {
142142
builder.setWeight(weight);
143143
}
144144

145+
@Override
146+
public int getMaxParallelApps() {
147+
QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
148+
return (p.hasMaxParallelApps()) ? p.getMaxParallelApps() : -1;
149+
}
150+
151+
@Override
152+
public void setMaxParallelApps(int weight) {
153+
maybeInitBuilder();
154+
builder.setMaxParallelApps(weight);
155+
}
156+
145157
@Override
146158
public void setChildQueues(List<QueueInfo> childQueues) {
147159
if (childQueues == null) {

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,7 @@ public static void setup() throws Exception {
435435
typeValueCache.put(QueueInfo.class, QueueInfo.
436436
newInstance("root", "root", 1.0f,
437437
1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"),
438-
"x && y", null, false, -1.0f, null, false));
438+
"x && y", null, false, -1.0f, 10, null, false));
439439
generateByNewInstance(QueueStatistics.class);
440440
generateByNewInstance(QueueUserACLInfo.class);
441441
generateByNewInstance(YarnClusterMetrics.class);

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,7 @@ protected QueueInfo getQueueInfo() {
774774
getIntraQueuePreemptionDisabled());
775775
queueInfo.setQueueConfigurations(getQueueConfigurations());
776776
queueInfo.setWeight(queueCapacities.getWeight());
777+
queueInfo.setMaxParallelApps(maxParallelApps);
777778
return queueInfo;
778779
}
779780

@@ -1494,6 +1495,7 @@ public void setMaxParallelApps(int maxParallelApps) {
14941495
this.maxParallelApps = maxParallelApps;
14951496
}
14961497

1498+
@Override
14971499
public int getMaxParallelApps() {
14981500
return maxParallelApps;
14991501
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,12 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
151151
* @return current run-state
152152
*/
153153
public QueueState getState();
154+
155+
/**
156+
* Get the max-parallel-applications property of the queue
157+
* @return max-parallel-applications
158+
*/
159+
public int getMaxParallelApps();
154160

155161
/**
156162
* Get child queues

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public class CapacitySchedulerQueueInfo {
6565
protected float weight;
6666
protected float normalizedWeight;
6767
protected int numApplications;
68+
protected int maxParallelApps;
6869
protected String queueName;
6970
protected boolean isAbsoluteResource;
7071
protected QueueState state;
@@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo {
120121
weight = q.getQueueCapacities().getWeight();
121122
normalizedWeight = q.getQueueCapacities().getNormalizedWeight();
122123
numApplications = q.getNumApplications();
124+
maxParallelApps = q.getMaxParallelApps();
123125
allocatedContainers = q.getMetrics().getAllocatedContainers();
124126
pendingContainers = q.getMetrics().getPendingContainers();
125127
reservedContainers = q.getMetrics().getReservedContainers();
@@ -352,6 +354,10 @@ public float getNormalizedWeight() {
352354
return normalizedWeight;
353355
}
354356

357+
public int getMaxParallelApps() {
358+
return maxParallelApps;
359+
}
360+
355361
public String getDefaultNodeLabelExpression() {
356362
return defaultNodeLabelExpression;
357363
}

0 commit comments

Comments
 (0)