YARN-10274. Merge QueueMapping and QueueMappingEntity. Contributed by Gergely Pollak

Author: szilard-nemeth

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java

@@ -48,15 +48,15 @@ public class AppNameMappingPlacementRule extends PlacementRule {
   private static final String QUEUE_MAPPING_NAME = "app-name";
 
   private boolean overrideWithQueueMappings = false;
-  private List<QueueMappingEntity> mappings = null;
+  private List<QueueMapping> mappings = null;
   protected CapacitySchedulerQueueManager queueManager;
 
   public AppNameMappingPlacementRule() {
     this(false, null);
   }
 
   public AppNameMappingPlacementRule(boolean overrideWithQueueMappings,
-      List<QueueMappingEntity> newMappings) {
+      List<QueueMapping> newMappings) {
     this.overrideWithQueueMappings = overrideWithQueueMappings;
     this.mappings = newMappings;
   }
@@ -76,16 +76,16 @@ public boolean initialize(ResourceScheduler scheduler)
     LOG.info(
         "Initialized App Name queue mappings, override: " + overrideWithQueueMappings);
 
-    List<QueueMappingEntity> queueMappings =
+    List<QueueMapping> queueMappings =
         conf.getQueueMappingEntity(QUEUE_MAPPING_NAME);
 
     // Get new user mappings
-    List<QueueMappingEntity> newMappings = new ArrayList<>();
+    List<QueueMapping> newMappings = new ArrayList<>();
 
     queueManager = schedulerContext.getCapacitySchedulerQueueManager();
 
     // check if mappings refer to valid queues
-    for (QueueMappingEntity mapping : queueMappings) {
+    for (QueueMapping mapping : queueMappings) {
       QueuePath queuePath = mapping.getQueuePath();
 
       if (isStaticQueueMapping(mapping)) {
@@ -109,7 +109,7 @@ public boolean initialize(ResourceScheduler scheduler)
           //validate if parent queue is specified,
           // then it should exist and
           // be an instance of AutoCreateEnabledParentQueue
-          QueueMappingEntity newMapping =
+          QueueMapping newMapping =
               validateAndGetAutoCreatedQueueMapping(queueManager, mapping,
                   queuePath);
           if (newMapping == null) {
@@ -123,7 +123,7 @@ public boolean initialize(ResourceScheduler scheduler)
           //   if its an instance of leaf queue
           //   if its an instance of auto created leaf queue,
           // then extract parent queue name and update queue mapping
-          QueueMappingEntity newMapping = validateAndGetQueueMapping(
+          QueueMapping newMapping = validateAndGetQueueMapping(
               queueManager, queue, mapping, queuePath);
           newMappings.add(newMapping);
         }
@@ -134,7 +134,7 @@ public boolean initialize(ResourceScheduler scheduler)
         // if parent queue is specified, then
         //  parent queue exists and an instance of AutoCreateEnabledParentQueue
         //
-        QueueMappingEntity newMapping = validateAndGetAutoCreatedQueueMapping(
+        QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
             queueManager, mapping, queuePath);
         if (newMapping != null) {
           newMappings.add(newMapping);
@@ -160,7 +160,7 @@ private static boolean ifQueueDoesNotExist(CSQueue queue) {
 
   private ApplicationPlacementContext getAppPlacementContext(String user,
       String applicationName) throws IOException {
-    for (QueueMappingEntity mapping : mappings) {
+    for (QueueMapping mapping : mappings) {
       if (mapping.getSource().equals(CURRENT_APP_MAPPING)) {
         if (mapping.getQueue().equals(CURRENT_APP_MAPPING)) {
           return getPlacementContext(mapping, applicationName, queueManager);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java

@@ -82,15 +82,17 @@ private QueueMapping(QueueMappingBuilder builder) {
     this.source = builder.source;
     this.queue = builder.queue;
     this.parentQueue = builder.parentQueue;
+    this.fullPath = (parentQueue != null) ? (parentQueue + DOT + queue) : queue;
   }
 
   /**
    * Different types of mapping.
    *
    */
   public enum MappingType {
-
-    USER("u"), GROUP("g");
+    USER("u"),
+    GROUP("g"),
+    APPLICATION("a");
 
     private final String type;
 
@@ -108,6 +110,7 @@ public String toString() {
   private String source;
   private String queue;
   private String parentQueue;
+  private String fullPath;
 
   private final static String DELIMITER = ":";
 
@@ -132,7 +135,7 @@ public String getSource() {
   }
 
   public String getFullPath() {
-    return (parentQueue != null ? parentQueue + DOT + queue : queue);
+    return fullPath;
   }
 
   public QueuePath getQueuePath() {
@@ -197,4 +200,10 @@ public String toString() {
     return type.toString() + DELIMITER + source + DELIMITER
         + (parentQueue != null ? parentQueue + "." + queue : queue);
   }
+
+  public String toTypelessString() {
+    return source + DELIMITER
+        + (parentQueue != null ? parentQueue + "." + queue : queue);
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java

@@ -65,25 +65,28 @@ public static void validateQueueMappingUnderParentQueue(
     }
   }
 
-  public static QueueMappingEntity validateAndGetAutoCreatedQueueMapping(
-      CapacitySchedulerQueueManager queueManager, QueueMappingEntity mapping,
+  public static QueueMapping validateAndGetAutoCreatedQueueMapping(
+      CapacitySchedulerQueueManager queueManager, QueueMapping mapping,
       QueuePath queuePath) throws IOException {
     if (queuePath.hasParentQueue()) {
       //if parent queue is specified,
       // then it should exist and be an instance of ManagedParentQueue
       validateQueueMappingUnderParentQueue(queueManager.getQueue(
           queuePath.getParentQueue()), queuePath.getParentQueue(),
           queuePath.getFullPath());
-      return new QueueMappingEntity(mapping.getSource(),
-          queuePath.getFullPath(), queuePath.getParentQueue());
+      return QueueMapping.QueueMappingBuilder.create()
+          .type(mapping.getType())
+          .source(mapping.getSource())
+          .queuePath(queuePath)
+          .build();
     }
 
     return null;
   }
 
-  public static QueueMappingEntity validateAndGetQueueMapping(
+  public static QueueMapping validateAndGetQueueMapping(
       CapacitySchedulerQueueManager queueManager, CSQueue queue,
-      QueueMappingEntity mapping, QueuePath queuePath) throws IOException {
+      QueueMapping mapping, QueuePath queuePath) throws IOException {
     if (!(queue instanceof LeafQueue)) {
       throw new IOException(
           "mapping contains invalid or non-leaf queue : " +
@@ -93,7 +96,7 @@ public static QueueMappingEntity validateAndGetQueueMapping(
     if (queue instanceof AutoCreatedLeafQueue && queue
         .getParent() instanceof ManagedParentQueue) {
 
-      QueueMappingEntity newMapping = validateAndGetAutoCreatedQueueMapping(
+      QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
           queueManager, mapping, queuePath);
       if (newMapping == null) {
         throw new IOException(
@@ -105,7 +108,7 @@ public static QueueMappingEntity validateAndGetQueueMapping(
     return mapping;
   }
 
-  public static boolean isStaticQueueMapping(QueueMappingEntity mapping) {
+  public static boolean isStaticQueueMapping(QueueMapping mapping) {
     return !mapping.getQueue().contains(CURRENT_USER_MAPPING) && !mapping
         .getQueue().contains(PRIMARY_GROUP_MAPPING)
         && !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING);
@@ -126,13 +129,13 @@ public static QueuePath extractQueuePath(String queuePath) {
   }
 
   public static ApplicationPlacementContext getPlacementContext(
-      QueueMappingEntity mapping, CapacitySchedulerQueueManager queueManager)
+      QueueMapping mapping, CapacitySchedulerQueueManager queueManager)
       throws IOException {
     return getPlacementContext(mapping, mapping.getQueue(), queueManager);
   }
 
   public static ApplicationPlacementContext getPlacementContext(
-      QueueMappingEntity mapping, String leafQueueName,
+      QueueMapping mapping, String leafQueueName,
       CapacitySchedulerQueueManager queueManager) throws IOException {
 
     //leafQueue name no longer identifies a queue uniquely checking ambiguity

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java

@@ -42,7 +42,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
@@ -1039,12 +1038,12 @@ public void setOverrideWithQueueMappings(boolean overrideWithQueueMappings) {
     setBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, overrideWithQueueMappings);
   }
 
-  public List<QueueMappingEntity> getQueueMappingEntity(
+  public List<QueueMapping> getQueueMappingEntity(
       String queueMappingSuffix) {
     String queueMappingName = buildQueueMappingRuleProperty(queueMappingSuffix);
 
-    List<QueueMappingEntity> mappings =
-        new ArrayList<QueueMappingEntity>();
+    List<QueueMapping> mappings =
+        new ArrayList<QueueMapping>();
     Collection<String> mappingsString =
         getTrimmedStringCollection(queueMappingName);
     for (String mappingValue : mappingsString) {
@@ -1058,10 +1057,11 @@ public List<QueueMappingEntity> getQueueMappingEntity(
 
       //Mappings should be consistent, and have the parent path parsed
       // from the beginning
-      QueueMappingEntity m = new QueueMappingEntity(
-          mapping[0],
-          QueuePlacementRuleUtils.extractQueuePath(mapping[1]));
-
+      QueueMapping m = QueueMapping.QueueMappingBuilder.create()
+          .type(QueueMapping.MappingType.APPLICATION)
+          .source(mapping[0])
+          .queuePath(QueuePlacementRuleUtils.extractQueuePath(mapping[1]))
+          .build();
       mappings.add(m);
     }
 
@@ -1076,15 +1076,15 @@ private String buildQueueMappingRuleProperty (String queueMappingSuffix) {
   }
 
   @VisibleForTesting
-  public void setQueueMappingEntities(List<QueueMappingEntity> queueMappings,
+  public void setQueueMappingEntities(List<QueueMapping> queueMappings,
       String queueMappingSuffix) {
     if (queueMappings == null) {
       return;
     }
 
     List<String> queueMappingStrs = new ArrayList<>();
-    for (QueueMappingEntity mapping : queueMappings) {
-      queueMappingStrs.add(mapping.toString());
+    for (QueueMapping mapping : queueMappings) {
+      queueMappingStrs.add(mapping.toTypelessString());
     }
 
     String mappingRuleProp = buildQueueMappingRuleProperty(queueMappingSuffix);