Skip to content

Commit 4046751

Browse files
p-szucsp-szucs
authored
YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup (#6499)
1 parent 897f446 commit 4046751

File tree

130 files changed

+2718
-2279
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

130 files changed

+2718
-2279
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
4646
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
4747
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
48+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
4849
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
4950
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
5051
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
@@ -180,9 +181,9 @@ public String getName() {
180181

181182
private static void setupQueueConfiguration(
182183
CapacitySchedulerConfiguration config) {
183-
config.setQueues(CapacitySchedulerConfiguration.ROOT,
184+
config.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT),
184185
new String[]{"testqueue"});
185-
String a = CapacitySchedulerConfiguration.ROOT + ".testqueue";
186+
QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".testqueue");
186187
config.setCapacity(a, 100f);
187188
config.setMaximumCapacity(a, 100f);
188189
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
4545
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
4646
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
47+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
4748
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
4849
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
4950
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
@@ -250,13 +251,13 @@ public void testCallRM() {
250251
CapacitySchedulerConfiguration csConf =
251252
new CapacitySchedulerConfiguration();
252253

253-
final String a = CapacitySchedulerConfiguration.ROOT + ".a";
254-
final String b = CapacitySchedulerConfiguration.ROOT + ".b";
255-
final String a1 = a + ".a1";
256-
final String a2 = a + ".a2";
257-
final String b1 = b + ".b1";
258-
final String b2 = b + ".b2";
259-
final String b3 = b + ".b3";
254+
final QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a");
255+
final QueuePath b = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".b");
256+
final QueuePath a1 = new QueuePath(a + ".a1");
257+
final QueuePath a2 = new QueuePath(a + ".a2");
258+
final QueuePath b1 = new QueuePath(b + ".b1");
259+
final QueuePath b2 = new QueuePath(b + ".b2");
260+
final QueuePath b3 = new QueuePath(b + ".b3");
260261
float aCapacity = 10.5f;
261262
float bCapacity = 89.5f;
262263
float a1Capacity = 30;
@@ -266,7 +267,7 @@ public void testCallRM() {
266267
float b3Capacity = 20;
267268

268269
// Define top-level queues
269-
csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
270+
csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT),
270271
new String[] {"a", "b"});
271272

272273
csConf.setCapacity(a, aCapacity);

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
5050
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
5151
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
52+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
5253
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
5354
import org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager;
5455
import org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager;
@@ -430,7 +431,7 @@ protected Plan initializePlan(String planQueueName) throws YarnException {
430431
Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy,
431432
getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation,
432433
maxAllocation, planQueueName, getReplanner(planQueuePath),
433-
getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath),
434+
getReservationSchedulerConfiguration().getMoveOnExpiry(new QueuePath(planQueuePath)),
434435
maxPeriodicity, rmContext);
435436
LOG.info("Initialized plan {} based on reservable queue {}",
436437
plan.toString(), planQueueName);
@@ -440,7 +441,7 @@ maxAllocation, planQueueName, getReplanner(planQueuePath),
440441
protected Planner getReplanner(String planQueueName) {
441442
ReservationSchedulerConfiguration reservationConfig =
442443
getReservationSchedulerConfiguration();
443-
String plannerClassName = reservationConfig.getReplanner(planQueueName);
444+
String plannerClassName = reservationConfig.getReplanner(new QueuePath(planQueueName));
444445
LOG.info("Using Replanner: " + plannerClassName + " for queue: "
445446
+ planQueueName);
446447
try {
@@ -463,7 +464,7 @@ protected Planner getReplanner(String planQueueName) {
463464
protected ReservationAgent getAgent(String queueName) {
464465
ReservationSchedulerConfiguration reservationConfig =
465466
getReservationSchedulerConfiguration();
466-
String agentClassName = reservationConfig.getReservationAgent(queueName);
467+
String agentClassName = reservationConfig.getReservationAgent(new QueuePath(queueName));
467468
LOG.info("Using Agent: " + agentClassName + " for queue: " + queueName);
468469
try {
469470
Class<?> agentClazz = conf.getClassByName(agentClassName);
@@ -487,7 +488,7 @@ protected SharingPolicy getAdmissionPolicy(String queueName) {
487488
ReservationSchedulerConfiguration reservationConfig =
488489
getReservationSchedulerConfiguration();
489490
String admissionPolicyClassName =
490-
reservationConfig.getReservationAdmissionPolicy(queueName);
491+
reservationConfig.getReservationAdmissionPolicy(new QueuePath(queueName));
491492
LOG.info("Using AdmissionPolicy: " + admissionPolicyClassName
492493
+ " for queue: " + queueName);
493494
try {

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
2424
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
2525
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
26+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
2627
import org.apache.hadoop.yarn.util.resource.Resources;
2728

2829
import java.util.Map;
@@ -57,9 +58,10 @@ public class CapacityOverTimePolicy extends NoOverCommitPolicy {
5758
private float maxAvg;
5859

5960
@Override
60-
public void init(String reservationQueuePath,
61+
public void init(String reservationQueue,
6162
ReservationSchedulerConfiguration conf) {
6263
this.conf = conf;
64+
QueuePath reservationQueuePath = new QueuePath(reservationQueue);
6365
validWindow = this.conf.getReservationWindow(reservationQueuePath);
6466
maxInst = this.conf.getInstantaneousMaxCapacity(reservationQueuePath) / 100;
6567
maxAvg = this.conf.getAverageCapacity(reservationQueuePath) / 100;

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.hadoop.security.authorize.AccessControlList;
2424
import org.apache.hadoop.yarn.api.records.ReservationACL;
2525
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
26+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
2627

2728
import java.util.Map;
2829

@@ -69,7 +70,7 @@ public ReservationSchedulerConfiguration(
6970
* @param queue name of the queue
7071
* @return true if the queue participates in reservation based scheduling
7172
*/
72-
public abstract boolean isReservable(String queue);
73+
public abstract boolean isReservable(QueuePath queue);
7374

7475
/**
7576
* Gets a map containing the {@link AccessControlList} of users for each
@@ -80,7 +81,7 @@ public ReservationSchedulerConfiguration(
8081
* which contains a list of users that have the specified permission level.
8182
*/
8283
public abstract Map<ReservationACL, AccessControlList> getReservationAcls(
83-
String queue);
84+
QueuePath queue);
8485

8586
/**
8687
* Gets the length of time in milliseconds for which the {@link SharingPolicy}
@@ -89,7 +90,7 @@ public abstract Map<ReservationACL, AccessControlList> getReservationAcls(
8990
* @return length in time in milliseconds for which to check the
9091
* {@link SharingPolicy}
9192
*/
92-
public long getReservationWindow(String queue) {
93+
public long getReservationWindow(QueuePath queue) {
9394
return DEFAULT_RESERVATION_WINDOW;
9495
}
9596

@@ -100,7 +101,7 @@ public long getReservationWindow(String queue) {
100101
* @param queue name of the queue
101102
* @return average capacity allowed by the {@link SharingPolicy}
102103
*/
103-
public float getAverageCapacity(String queue) {
104+
public float getAverageCapacity(QueuePath queue) {
104105
return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
105106
}
106107

@@ -109,7 +110,7 @@ public float getAverageCapacity(String queue) {
109110
* @param queue name of the queue
110111
* @return maximum allowed capacity at any time
111112
*/
112-
public float getInstantaneousMaxCapacity(String queue) {
113+
public float getInstantaneousMaxCapacity(QueuePath queue) {
113114
return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
114115
}
115116

@@ -118,7 +119,7 @@ public float getInstantaneousMaxCapacity(String queue) {
118119
* @param queue name of the queue
119120
* @return the class name of the {@link SharingPolicy}
120121
*/
121-
public String getReservationAdmissionPolicy(String queue) {
122+
public String getReservationAdmissionPolicy(QueuePath queue) {
122123
return DEFAULT_RESERVATION_ADMISSION_POLICY;
123124
}
124125

@@ -128,7 +129,7 @@ public String getReservationAdmissionPolicy(String queue) {
128129
* @param queue name of the queue
129130
* @return the class name of the {@code ReservationAgent}
130131
*/
131-
public String getReservationAgent(String queue) {
132+
public String getReservationAgent(QueuePath queue) {
132133
return DEFAULT_RESERVATION_AGENT_NAME;
133134
}
134135

@@ -137,7 +138,7 @@ public String getReservationAgent(String queue) {
137138
* @param queuePath name of the queue
138139
* @return true if reservation queues should be visible
139140
*/
140-
public boolean getShowReservationAsQueues(String queuePath) {
141+
public boolean getShowReservationAsQueues(QueuePath queuePath) {
141142
return DEFAULT_SHOW_RESERVATIONS_AS_QUEUES;
142143
}
143144

@@ -147,7 +148,7 @@ public boolean getShowReservationAsQueues(String queuePath) {
147148
* @param queue name of the queue
148149
* @return the class name of the {@code Planner}
149150
*/
150-
public String getReplanner(String queue) {
151+
public String getReplanner(QueuePath queue) {
151152
return DEFAULT_RESERVATION_PLANNER_NAME;
152153
}
153154

@@ -158,7 +159,7 @@ public String getReplanner(String queue) {
158159
* @return true if application should be moved, false if they need to be
159160
* killed
160161
*/
161-
public boolean getMoveOnExpiry(String queue) {
162+
public boolean getMoveOnExpiry(QueuePath queue) {
162163
return DEFAULT_RESERVATION_MOVE_ON_EXPIRY;
163164
}
164165

@@ -168,7 +169,7 @@ public boolean getMoveOnExpiry(String queue) {
168169
* @param queue name of the queue
169170
* @return the time in milliseconds for which to check constraints
170171
*/
171-
public long getEnforcementWindow(String queue) {
172+
public long getEnforcementWindow(QueuePath queue) {
172173
return DEFAULT_RESERVATION_ENFORCEMENT_WINDOW;
173174
}
174175
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.Set;
2424
import java.util.TreeSet;
2525

26+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
2627
import org.slf4j.Logger;
2728
import org.slf4j.LoggerFactory;
2829
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
@@ -72,7 +73,7 @@ public SimpleCapacityReplanner() {
7273
@Override
7374
public void init(String planQueueName,
7475
ReservationSchedulerConfiguration conf) {
75-
this.lengthOfCheckZone = conf.getEnforcementWindow(planQueueName);
76+
this.lengthOfCheckZone = conf.getEnforcementWindow(new QueuePath(planQueueName));
7677
}
7778

7879
@Override

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -61,26 +61,26 @@ public void setEntitlement(QueueEntitlement entitlement)
6161
}
6262

6363
@Override
64-
protected Resource getMinimumAbsoluteResource(String queuePath,
64+
protected Resource getMinimumAbsoluteResource(QueuePath queuePath,
6565
String label) {
66-
return super.getMinimumAbsoluteResource(queueContext.getConfiguration()
67-
.getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
66+
return super.getMinimumAbsoluteResource(QueuePrefixes
67+
.getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
6868
label);
6969
}
7070

7171
@Override
72-
protected Resource getMaximumAbsoluteResource(String queuePath,
72+
protected Resource getMaximumAbsoluteResource(QueuePath queuePath,
7373
String label) {
74-
return super.getMaximumAbsoluteResource(queueContext.getConfiguration()
75-
.getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
74+
return super.getMaximumAbsoluteResource(QueuePrefixes
75+
.getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
7676
label);
7777
}
7878

7979
@Override
80-
protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
80+
protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath,
8181
String label) {
82-
return super.checkConfigTypeIsAbsoluteResource(queueContext.getConfiguration()
83-
.getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
82+
return super.checkConfigTypeIsAbsoluteResource(QueuePrefixes
83+
.getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
8484
label);
8585
}
8686

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws
346346
writeLock.lock();
347347
try {
348348
CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
349-
this.acls = configuration.getAcls(getQueuePath());
349+
this.acls = configuration.getAcls(getQueuePathObject());
350350

351351
if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
352352
parseAndSetDynamicTemplates();
@@ -367,7 +367,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws
367367

368368
// Setup queue's maximumAllocation respecting the global
369369
// and the queue settings
370-
this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(),
370+
this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePathObject(),
371371
parent);
372372

373373
// Initialize the queue state based on previous state, configured state
@@ -382,10 +382,10 @@ protected void setupQueueConfigs(Resource clusterResource) throws
382382
configuration.getReservationContinueLook();
383383

384384
this.configuredCapacityVectors = configuration
385-
.parseConfiguredResourceVector(queuePath.getFullPath(),
385+
.parseConfiguredResourceVector(queuePath,
386386
this.queueNodeLabelsSettings.getConfiguredNodeLabels());
387387
this.configuredMaxCapacityVectors = configuration
388-
.parseConfiguredMaximumCapacityVector(queuePath.getFullPath(),
388+
.parseConfiguredMaximumCapacityVector(queuePath,
389389
this.queueNodeLabelsSettings.getConfiguredNodeLabels(),
390390
QueueCapacityVector.newInstance());
391391

@@ -420,11 +420,11 @@ protected void setupQueueConfigs(Resource clusterResource) throws
420420
// Store preemption settings
421421
this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration);
422422
this.priority = configuration.getQueuePriority(
423-
getQueuePath());
423+
getQueuePathObject());
424424

425425
// Update multi-node sorting algorithm for scheduling as configured.
426426
setMultiNodeSortingPolicyName(
427-
configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath()));
427+
configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePathObject()));
428428

429429
// Setup application related limits
430430
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
@@ -440,7 +440,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws
440440
protected void parseAndSetDynamicTemplates() {
441441
// Set the template properties from the parent to the queuepath of the child
442442
((AbstractParentQueue) parent).getAutoCreatedQueueTemplate()
443-
.setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath(),
443+
.setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePathObject(),
444444
this instanceof AbstractLeafQueue);
445445

446446
String parentTemplate = String.format("%s.%s", parent.getQueuePath(),
@@ -488,21 +488,21 @@ private UserWeights getUserWeightsFromHierarchy() {
488488
// Insert this queue's userWeights, overriding parent's userWeights if
489489
// there is an overlap.
490490
unionInheritedWeights.addFrom(
491-
queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePath()));
491+
queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePathObject()));
492492
return unionInheritedWeights;
493493
}
494494

495-
protected Resource getMinimumAbsoluteResource(String queuePath, String label) {
495+
protected Resource getMinimumAbsoluteResource(QueuePath queuePath, String label) {
496496
return queueContext.getConfiguration()
497497
.getMinimumResourceRequirement(label, queuePath, resourceTypes);
498498
}
499499

500-
protected Resource getMaximumAbsoluteResource(String queuePath, String label) {
500+
protected Resource getMaximumAbsoluteResource(QueuePath queuePath, String label) {
501501
return queueContext.getConfiguration()
502502
.getMaximumResourceRequirement(label, queuePath, resourceTypes);
503503
}
504504

505-
protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
505+
protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath,
506506
String label) {
507507
return queueContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label,
508508
queuePath, resourceTypes);
@@ -518,7 +518,7 @@ protected void updateCapacityConfigType() {
518518

519519
if (queueContext.getConfiguration().isLegacyQueueMode()) {
520520
localType = checkConfigTypeIsAbsoluteResource(
521-
getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE
521+
getQueuePathObject(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE
522522
: CapacityConfigType.PERCENTAGE;
523523
} else {
524524
// TODO: revisit this later
@@ -556,8 +556,8 @@ protected void updateCapacityConfigType() {
556556
*/
557557
protected void updateConfigurableResourceLimits(Resource clusterResource) {
558558
for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
559-
final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label);
560-
Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label);
559+
final Resource minResource = getMinimumAbsoluteResource(getQueuePathObject(), label);
560+
Resource maxResource = getMaximumAbsoluteResource(getQueuePathObject(), label);
561561

562562
if (parent != null) {
563563
final Resource parentMax = parent.getQueueResourceQuotas()

0 commit comments

Comments
 (0)