From 9d92656bd2da885b3d7eab0bc04832679373f541 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Fri, 9 Dec 2022 16:09:02 +0530 Subject: [PATCH 1/5] added default ulf to -1 --- .../converter/FSConfigToCSConfigConverter.java | 10 ++++++++++ .../TestFSConfigToCSConfigConverter.java | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 25832c7633bf1..6ec53d804bf14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED; import java.io.ByteArrayOutputStream; @@ -295,6 +296,7 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { emitDefaultUserMaxParallelApplications(); emitUserMaxParallelApplications(); emitDefaultMaxAMShare(); + emitDefaultUserLimitFactor(); emitDisablePreemptionForObserveOnlyMode(); FSQueueConverter queueConverter = FSQueueConverterBuilder.create() @@ -412,6 +414,14 @@ private void emitDefaultMaxAMShare() { queueMaxAMShareDefault); } } + + private void emitDefaultUserLimitFactor() { + capacitySchedulerConfig.setFloat( + CapacitySchedulerConfiguration. + PREFIX + USER_LIMIT_FACTOR, + -1.0f); + } + private void emitDisablePreemptionForObserveOnlyMode() { if (preemptionMode == FSConfigToCSConfigConverterParams .PreemptionMode.OBSERVE_ONLY) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index c1c774bed0942..80f80cabc710d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; @@ -185,6 +186,20 @@ public void testDefaultMaxAMShare() throws Exception { conf.get(PREFIX + "root.users.joe maximum-am-resource-percent")); } + @Test + public void testDefaultUserLimitFactor() throws Exception { + converter.convert(config); + + Configuration conf = converter.getCapacitySchedulerConfig(); + String userLimitFactor = + conf.get(PREFIX + USER_LIMIT_FACTOR); + + assertEquals("Default user limit factor", "-1.0", userLimitFactor); + + assertNull("root.users.joe user-limit-factor should be null", + conf.get(PREFIX + "root.users.joe user-limit-factor")); + } + @Test public void testDefaultMaxAMShareDisabled() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() From 27aa5f8cdbd4e1a481bf6df2d3375c69a6dbb34a Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Wed, 14 Dec 2022 18:58:20 +0530 Subject: [PATCH 2/5] no global user-limit-factor, adding one for each leafqueue Change-Id: Ia576cf1c71fee2b80725f2e60f91cd3174640653 --- .../scheduler/fair/converter/FSConfigToCSConfigConverter.java | 2 +- .../fair/converter/TestFSConfigToCSConfigConverter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 6ec53d804bf14..96f5c639fde5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -418,7 +418,7 @@ private void emitDefaultMaxAMShare() { private void emitDefaultUserLimitFactor() { capacitySchedulerConfig.setFloat( CapacitySchedulerConfiguration. - PREFIX + USER_LIMIT_FACTOR, + PREFIX + "root.default." + USER_LIMIT_FACTOR, -1.0f); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 80f80cabc710d..f9dfcc2ece4c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -192,7 +192,7 @@ public void testDefaultUserLimitFactor() throws Exception { Configuration conf = converter.getCapacitySchedulerConfig(); String userLimitFactor = - conf.get(PREFIX + USER_LIMIT_FACTOR); + conf.get(PREFIX + "root.default." + USER_LIMIT_FACTOR); assertEquals("Default user limit factor", "-1.0", userLimitFactor); From b60583844c5ee17f6e89f20006313798890918e0 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Thu, 22 Dec 2022 21:19:44 +0530 Subject: [PATCH 3/5] added ulf to -1 only for those queues in which auto-queue-creation is disabled Change-Id: I99f62b9c4f40027689f2953bb4be946e23261ae5 --- .../FSConfigToCSConfigConverter.java | 22 ++++++++++++++----- .../TestFSConfigToCSConfigConverter.java | 9 +++++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 96f5c639fde5d..634a999932d4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -16,6 +16,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON; @@ -30,6 +31,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -49,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -292,11 +295,11 @@ private void convertYarnSiteXml(Configuration inputYarnSiteConfig) { private void convertCapacitySchedulerXml(FairScheduler fs) { FSParentQueue rootQueue = fs.getQueueManager().getRootQueue(); + Collection fsLeafQueue = fs.getQueueManager().getLeafQueues(); emitDefaultQueueMaxParallelApplications(); emitDefaultUserMaxParallelApplications(); emitUserMaxParallelApplications(); emitDefaultMaxAMShare(); - emitDefaultUserLimitFactor(); emitDisablePreemptionForObserveOnlyMode(); FSQueueConverter queueConverter = FSQueueConverterBuilder.create() @@ -312,6 +315,7 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { .withPercentages(usePercentages) .build(); + emitDefaultUserLimitFactor(fsLeafQueue); queueConverter.convertQueueHierarchy(rootQueue); emitACLs(fs); } @@ -415,11 +419,17 @@ private void emitDefaultMaxAMShare() { } } - private void emitDefaultUserLimitFactor() { - capacitySchedulerConfig.setFloat( - CapacitySchedulerConfiguration. - PREFIX + "root.default." + USER_LIMIT_FACTOR, - -1.0f); + private void emitDefaultUserLimitFactor(Collection fsLeafQueue) { + fsLeafQueue + .forEach((leafQueue) -> { + if (!capacitySchedulerConfig. + isAutoQueueCreationV2Enabled(leafQueue.getName())) { + capacitySchedulerConfig.setFloat( + CapacitySchedulerConfiguration. + PREFIX + leafQueue.getName() + DOT + USER_LIMIT_FACTOR, + -1.0f); + } + }); } private void emitDisablePreemptionForObserveOnlyMode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index f9dfcc2ece4c8..4bd2058b6ec25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -183,7 +183,7 @@ public void testDefaultMaxAMShare() throws Exception { conf.get(PREFIX + "root.admins.alice.maximum-am-resource-percent")); assertNull("root.users.joe maximum-am-resource-percent should be null", - conf.get(PREFIX + "root.users.joe maximum-am-resource-percent")); + conf.get(PREFIX + "root.users.joe.maximum-am-resource-percent")); } @Test @@ -196,8 +196,11 @@ public void testDefaultUserLimitFactor() throws Exception { assertEquals("Default user limit factor", "-1.0", userLimitFactor); - assertNull("root.users.joe user-limit-factor should be null", - conf.get(PREFIX + "root.users.joe user-limit-factor")); + assertEquals("root.users.joe user-limit-factor", "-1.0", + conf.get(PREFIX + "root.users.joe.user-limit-factor")); + + assertEquals("root.admins.bob user-limit-factor", "-1.0", + conf.get(PREFIX + "root.admins.bob.user-limit-factor")); } @Test From 5489ce5e55796f0ea870aea613ee3ca58d42d150 Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Fri, 23 Dec 2022 14:58:03 +0530 Subject: [PATCH 4/5] moved method emitDefaultUserLimitFactor to FSqueueConverter Change-Id: Ife4762de43fd21924d2b481502580e74a8242d83 --- .../converter/FSConfigToCSConfigConverter.java | 17 +---------------- .../fair/converter/FSQueueConverter.java | 17 +++++++++++++++++ .../TestFSConfigToCSConfigConverter.java | 8 +++++--- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 634a999932d4e..34a7f3b321137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -16,12 +16,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED; import java.io.ByteArrayOutputStream; @@ -315,7 +313,7 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { .withPercentages(usePercentages) .build(); - emitDefaultUserLimitFactor(fsLeafQueue); + queueConverter.emitDefaultUserLimitFactor(fsLeafQueue, capacitySchedulerConfig); queueConverter.convertQueueHierarchy(rootQueue); emitACLs(fs); } @@ -419,19 +417,6 @@ private void emitDefaultMaxAMShare() { } } - private void emitDefaultUserLimitFactor(Collection fsLeafQueue) { - fsLeafQueue - .forEach((leafQueue) -> { - if (!capacitySchedulerConfig. - isAutoQueueCreationV2Enabled(leafQueue.getName())) { - capacitySchedulerConfig.setFloat( - CapacitySchedulerConfiguration. - PREFIX + leafQueue.getName() + DOT + USER_LIMIT_FACTOR, - -1.0f); - } - }); - } - private void emitDisablePreemptionForObserveOnlyMode() { if (preemptionMode == FSConfigToCSConfigConverterParams .PreemptionMode.OBSERVE_ONLY) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 0e9b389488544..4f00240b98875 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -17,12 +17,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import java.util.List; +import java.util.Collection; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -215,6 +219,19 @@ private void emitPreemptionDisabled(String queueName, FSQueue queue) { } } + public void emitDefaultUserLimitFactor(Collection fsLeafQueue, CapacitySchedulerConfiguration config) { + fsLeafQueue + .forEach((leafQueue) -> { + if (!config. + isAutoQueueCreationV2Enabled(leafQueue.getName())) { + config.setFloat( + CapacitySchedulerConfiguration. + PREFIX + leafQueue.getName() + DOT + USER_LIMIT_FACTOR, + -1.0f); + } + }); + } + /** * yarn.scheduler.fair.sizebasedweight ==> * yarn.scheduler.capacity.<queue-path> diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 4bd2058b6ec25..86bf113d64ed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -191,10 +191,12 @@ public void testDefaultUserLimitFactor() throws Exception { converter.convert(config); Configuration conf = converter.getCapacitySchedulerConfig(); - String userLimitFactor = - conf.get(PREFIX + "root.default." + USER_LIMIT_FACTOR); - assertEquals("Default user limit factor", "-1.0", userLimitFactor); + assertNull("root.users user-limit-factor should be null", + conf.get(PREFIX + "root.users." + USER_LIMIT_FACTOR)); + + assertEquals("root.default user-limit-factor", "-1.0", + conf.get(PREFIX + "root.default.user-limit-factor")); assertEquals("root.users.joe user-limit-factor", "-1.0", conf.get(PREFIX + "root.users.joe.user-limit-factor")); From e5d1b9c54df1189914efdf2c0292ca7dbbccd86f Mon Sep 17 00:00:00 2001 From: Susheel Gupta Date: Mon, 26 Dec 2022 12:08:38 +0530 Subject: [PATCH 5/5] fix of method call emitDefaultUserLimitFactor Change-Id: I79bdb749c2ce2db1b37cdc4813f6b8527d057fb0 --- .../FSConfigToCSConfigConverter.java | 5 ----- .../fair/converter/FSQueueConverter.java | 20 ++++++++----------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 34a7f3b321137..25832c7633bf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -49,7 +48,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -293,7 +291,6 @@ private void convertYarnSiteXml(Configuration inputYarnSiteConfig) { private void convertCapacitySchedulerXml(FairScheduler fs) { FSParentQueue rootQueue = fs.getQueueManager().getRootQueue(); - Collection fsLeafQueue = fs.getQueueManager().getLeafQueues(); emitDefaultQueueMaxParallelApplications(); emitDefaultUserMaxParallelApplications(); emitUserMaxParallelApplications(); @@ -313,7 +310,6 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { .withPercentages(usePercentages) .build(); - queueConverter.emitDefaultUserLimitFactor(fsLeafQueue, capacitySchedulerConfig); queueConverter.convertQueueHierarchy(rootQueue); emitACLs(fs); } @@ -416,7 +412,6 @@ private void emitDefaultMaxAMShare() { queueMaxAMShareDefault); } } - private void emitDisablePreemptionForObserveOnlyMode() { if (preemptionMode == FSConfigToCSConfigConverterParams .PreemptionMode.OBSERVE_ONLY) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 4f00240b98875..c29d020b10ce3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import java.util.List; -import java.util.Collection; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -83,6 +82,7 @@ public void convertQueueHierarchy(FSQueue queue) { emitMaxParallelApps(queueName, queue); emitMaxAllocations(queueName, queue); emitPreemptionDisabled(queueName, queue); + emitDefaultUserLimitFactor(queueName, children); emitChildCapacity(queue); emitMaximumCapacity(queueName, queue); @@ -219,17 +219,13 @@ private void emitPreemptionDisabled(String queueName, FSQueue queue) { } } - public void emitDefaultUserLimitFactor(Collection fsLeafQueue, CapacitySchedulerConfiguration config) { - fsLeafQueue - .forEach((leafQueue) -> { - if (!config. - isAutoQueueCreationV2Enabled(leafQueue.getName())) { - config.setFloat( - CapacitySchedulerConfiguration. - PREFIX + leafQueue.getName() + DOT + USER_LIMIT_FACTOR, - -1.0f); - } - }); + public void emitDefaultUserLimitFactor(String queueName, List children) { + if (children.isEmpty()) { + capacitySchedulerConfig.setFloat( + CapacitySchedulerConfiguration. + PREFIX + queueName + DOT + USER_LIMIT_FACTOR, + -1.0f); + } } /**