Skip to content

Commit 213d3de

Browse files
ericbadgerericbadger
committed
YARN-10503. Support queue capacity in terms of absolute resources with custom
resourceType. Contributed by Qi Zhu.
1 parent bf66116 commit 213d3de

File tree

4 files changed

+126
-8
lines changed

4 files changed

+126
-8
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,4 +902,19 @@ private static void validateResourceTypes(
902902
}
903903
}
904904
}
905+
906+
public static StringBuilder
907+
getCustomResourcesStrings(Resource resource) {
908+
StringBuilder res = new StringBuilder();
909+
if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
910+
ResourceInformation[] resources =
911+
resource.getResources();
912+
for (int i = 2; i < resources.length; i++) {
913+
ResourceInformation resInfo = resources[i];
914+
res.append(","
915+
+ resInfo.getName() + "=" + resInfo.getValue());
916+
}
917+
}
918+
return res;
919+
}
905920
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2350,11 +2350,14 @@ public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath,
23502350
getAutoCreatedQueueTemplateConfPrefix(queuePath);
23512351

23522352
StringBuilder resourceString = new StringBuilder();
2353+
23532354
resourceString
23542355
.append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
23552356
+ resource.getMemorySize() + ","
23562357
+ AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
2357-
+ resource.getVirtualCores() + "]");
2358+
+ resource.getVirtualCores()
2359+
+ ResourceUtils.
2360+
getCustomResourcesStrings(resource) + "]");
23582361

23592362
setCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString());
23602363
}
@@ -2385,11 +2388,14 @@ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
23852388
queuePath);
23862389

23872390
StringBuilder resourceString = new StringBuilder();
2391+
23882392
resourceString
23892393
.append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
23902394
+ resource.getMemorySize() + ","
23912395
+ AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
2392-
+ resource.getVirtualCores() + "]");
2396+
+ resource.getVirtualCores()
2397+
+ ResourceUtils.
2398+
getCustomResourcesStrings(resource) + "]");
23932399

23942400
setMaximumCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString());
23952401
}
@@ -2489,11 +2495,14 @@ private void updateMinMaxResourceToConf(String label, String queue,
24892495
}
24902496

24912497
StringBuilder resourceString = new StringBuilder();
2498+
24922499
resourceString
24932500
.append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
24942501
+ resource.getMemorySize() + ","
24952502
+ AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
2496-
+ resource.getVirtualCores() + "]");
2503+
+ resource.getVirtualCores()
2504+
+ ResourceUtils.
2505+
getCustomResourcesStrings(resource) + "]");
24972506

24982507
String prefix = getQueuePrefix(queue) + type;
24992508
if (!label.isEmpty()) {
@@ -2567,8 +2576,12 @@ private Resource internalGetLabeledResourceRequirementForQueue(String queue,
25672576
private void updateResourceValuesFromConfig(Set<String> resourceTypes,
25682577
Resource resource, String[] splits) {
25692578

2579+
String resourceName = splits[0].trim();
2580+
25702581
// If key is not a valid type, skip it.
2571-
if (!resourceTypes.contains(splits[0])) {
2582+
if (!resourceTypes.contains(resourceName)
2583+
&& !ResourceUtils.getResourceTypes().containsKey(resourceName)) {
2584+
LOG.error(resourceName + " not supported.");
25722585
return;
25732586
}
25742587

@@ -2581,9 +2594,17 @@ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
25812594
resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
25822595
}
25832596

2597+
// Custom resource type defined by user.
2598+
// Such as GPU FPGA etc.
2599+
if (!resourceTypes.contains(resourceName)) {
2600+
resource.setResourceInformation(resourceName, ResourceInformation
2601+
.newInstance(resourceName, units, resourceValue));
2602+
return;
2603+
}
2604+
25842605
// map it based on key.
25852606
AbsoluteResourceType resType = AbsoluteResourceType
2586-
.valueOf(StringUtils.toUpperCase(splits[0].trim()));
2607+
.valueOf(StringUtils.toUpperCase(resourceName));
25872608
switch (resType) {
25882609
case MEMORY :
25892610
resource.setMemorySize(resourceValue);
@@ -2592,8 +2613,8 @@ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
25922613
resource.setVirtualCores(resourceValue.intValue());
25932614
break;
25942615
default :
2595-
resource.setResourceInformation(splits[0].trim(), ResourceInformation
2596-
.newInstance(splits[0].trim(), units, resourceValue));
2616+
resource.setResourceInformation(resourceName, ResourceInformation
2617+
.newInstance(resourceName, units, resourceValue));
25972618
break;
25982619
}
25992620
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
2020

2121
import org.apache.commons.io.FileUtils;
22+
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
2223
import org.apache.hadoop.yarn.api.records.ContainerId;
2324
import org.apache.hadoop.yarn.api.records.Resource;
2425
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -50,10 +51,16 @@
5051

5152
import java.io.File;
5253
import java.io.IOException;
54+
5355
import java.util.ArrayList;
5456
import java.util.Collections;
5557
import java.util.List;
58+
import java.util.Set;
59+
import java.util.Map;
60+
import java.util.Arrays;
61+
import java.util.stream.Collectors;
5662

63+
import static org.apache.hadoop.yarn.api.records.ResourceInformation.FPGA_URI;
5764
import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
5865
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
5966
import static org.junit.Assert.assertEquals;
@@ -248,4 +255,79 @@ public void testClusterMetricsWithGPU()
248255
.get(GPU_URI)).longValue(), 0);
249256
ClusterMetrics.destroy();
250257
}
258+
259+
/**
260+
* Test CS absolute conf with Custom resource type.
261+
* */
262+
@Test
263+
public void testCapacitySchedulerAbsoluteConfWithCustomResourceType()
264+
throws IOException {
265+
// reset resource types
266+
ResourceUtils.resetResourceTypes();
267+
String resourceTypesFileName = "resource-types-test.xml";
268+
File source = new File(
269+
conf.getClassLoader().getResource(resourceTypesFileName).getFile());
270+
resourceTypesFile = new File(source.getParent(), "resource-types.xml");
271+
FileUtils.copyFile(source, resourceTypesFile);
272+
273+
CapacitySchedulerConfiguration newConf =
274+
new CapacitySchedulerConfiguration(conf);
275+
276+
// Only memory vcores for first class.
277+
Set<String> resourceTypes = Arrays.
278+
stream(CapacitySchedulerConfiguration.
279+
AbsoluteResourceType.values()).
280+
map(value -> value.toString().toLowerCase()).
281+
collect(Collectors.toSet());
282+
283+
Map<String, Long> valuesMin = Maps.newHashMap();
284+
valuesMin.put(GPU_URI, 10L);
285+
valuesMin.put(FPGA_URI, 10L);
286+
valuesMin.put("testType", 10L);
287+
288+
Map<String, Long> valuesMax = Maps.newHashMap();
289+
valuesMax.put(GPU_URI, 100L);
290+
valuesMax.put(FPGA_URI, 100L);
291+
valuesMax.put("testType", 100L);
292+
293+
Resource aMINRES =
294+
Resource.newInstance(1000, 10, valuesMin);
295+
296+
Resource aMAXRES =
297+
Resource.newInstance(1000, 10, valuesMax);
298+
299+
// Define top-level queues
300+
newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
301+
new String[] {"a", "b", "c"});
302+
newConf.setMinimumResourceRequirement("", "root.a",
303+
aMINRES);
304+
newConf.setMaximumResourceRequirement("", "root.a",
305+
aMAXRES);
306+
307+
newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
308+
DominantResourceCalculator.class, ResourceCalculator.class);
309+
310+
//start RM
311+
MockRM rm = new MockRM(newConf);
312+
rm.start();
313+
314+
// Check the gpu resource conf is right.
315+
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
316+
Assert.assertEquals(aMINRES,
317+
cs.getConfiguration().
318+
getMinimumResourceRequirement("", "root.a", resourceTypes));
319+
Assert.assertEquals(aMAXRES,
320+
cs.getConfiguration().
321+
getMaximumResourceRequirement("", "root.a", resourceTypes));
322+
323+
// Check the gpu resource of queue is right.
324+
Assert.assertEquals(aMINRES, cs.getQueue("root.a").
325+
getQueueResourceQuotas().getConfiguredMinResource());
326+
Assert.assertEquals(aMAXRES, cs.getQueue("root.a").
327+
getQueueResourceQuotas().getConfiguredMaxResource());
328+
329+
rm.close();
330+
331+
}
332+
251333
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@
1717
<configuration>
1818
<property>
1919
<name>yarn.resource-types</name>
20-
<value>yarn.io/gpu, yarn.io/fpga</value>
20+
<value>yarn.io/gpu, yarn.io/fpga, testType</value>
2121
</property>
2222
</configuration>

0 commit comments

Comments
 (0)