From 09897bc1fad474075615aea60d7a293a29ef553e Mon Sep 17 00:00:00 2001 From: skalva Date: Thu, 9 Mar 2023 14:37:19 +0530 Subject: [PATCH 1/4] NPE fix while accessing nodelables --- .../scheduler/capacity/QueueNodeLabelsSettings.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java index c431d2bb45543..2c9010f441b34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -53,9 +53,15 @@ private void initializeNodeLabels(CapacitySchedulerConfiguration configuration, } private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { - this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath()); - // Inherit labels from parent if not set - if (this.accessibleLabels == null && parent != null) { + Set nodeLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath()); + if (null != nodeLabels) { + // Reading "accessibleLabels" can cause NPE without ReadLock. + // Since getAccessibleNodeLabels() can return null, If someone access + // "accessibleLabels" before assigning it from parent can cause NPE. + // So use local instance and assign it only if not null + this.accessibleLabels = nodeLabels; + } else if (null != parent) { + // Inherit labels from parent if not set this.accessibleLabels = parent.getAccessibleNodeLabels(); } } From b9d4fca4dd447db65385cc89db59e4090e105744 Mon Sep 17 00:00:00 2001 From: skalva Date: Sun, 23 Apr 2023 12:39:34 +0530 Subject: [PATCH 2/4] Empty-Commit From 3a888f9f042cc9029d2ffb7aa50a503345dcfbdb Mon Sep 17 00:00:00 2001 From: skalva Date: Mon, 24 Apr 2023 11:49:17 +0530 Subject: [PATCH 3/4] unit test cases for YARN-11449 changes --- .../scheduler/capacity/TestLeafQueue.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index eca065b148766..74c2f760501e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -5431,4 +5431,23 @@ public void testSubmitUsingRealUserAcls() throws Exception { rm.stop(); rm.close(); } + + @Test + public void testLeafHasAllNodeLabelsOfItsRoot() throws IOException { + + CapacitySchedulerConfiguration conf = csConf; + String rootChild = root.getChildQueues().get(0).getQueuePath(); + ParentQueue rootQueue = (ParentQueue) cs.getRootQueue(); + ConfiguredNodeLabels nodeLabelsForAllQueues = + rootQueue.getQueueContext().getQueueManager().getConfiguredNodeLabelsForAllQueues(); + + QueueNodeLabelsSettings labelsSettings = + new QueueNodeLabelsSettings(conf, root, new QueuePath(rootQueue.getQueuePath(), "test"), nodeLabelsForAllQueues); + Assert.assertEquals(labelsSettings.getAccessibleNodeLabels(), "*"); + + labelsSettings = new QueueNodeLabelsSettings(conf, root, + new QueuePath(rootQueue.getQueuePath(), rootQueue.getChildQueues().get(0).getQueuePath()), + nodeLabelsForAllQueues); + Assert.assertEquals(labelsSettings.getAccessibleNodeLabels(), "*"); + } } From 01dda553f1a70f517720da550a1e9468820a8a19 Mon Sep 17 00:00:00 2001 From: skalva Date: Mon, 24 Apr 2023 12:59:06 +0530 Subject: [PATCH 4/4] Empty-Commit