From 12a0968fd7561edc259a7b175d29f76c56a97b8f Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Thu, 27 Feb 2020 18:05:06 +0100 Subject: [PATCH] YARN-9879 Allow multiple leaf queues with the same name in CS --- .../sls/scheduler/SLSCapacityScheduler.java | 2 +- .../CapacitySchedulerPreemptionUtils.java | 2 +- .../FifoIntraQueuePreemptionPlugin.java | 4 +- .../ProportionalCapacityPreemptionPolicy.java | 2 +- .../AppNameMappingPlacementRule.java | 78 ++++----- .../placement/QueueMapping.java | 17 ++ .../placement/QueueMappingEntity.java | 29 +++- .../resourcemanager/placement/QueuePath.java | 9 +- .../placement/QueuePlacementRuleUtils.java | 34 ++-- .../UserGroupMappingPlacementRule.java | 142 ++++++++-------- .../CapacityReservationSystem.java | 5 + .../scheduler/AbstractYarnScheduler.java | 13 +- .../SchedulerApplicationAttempt.java | 13 +- .../activities/ActivitiesManager.java | 10 ++ .../AbstractAutoCreatedLeafQueue.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 54 ++++--- .../capacity/AbstractManagedParentQueue.java | 2 +- .../capacity/AutoCreatedLeafQueue.java | 2 +- .../scheduler/capacity/CSQueue.java | 8 +- .../scheduler/capacity/CSQueueStore.java | 151 ++++++++++++++++++ .../scheduler/capacity/CapacityScheduler.java | 61 ++++--- .../CapacitySchedulerConfigValidator.java | 71 ++++---- .../CapacitySchedulerConfiguration.java | 10 +- .../CapacitySchedulerQueueManager.java | 120 +++++++------- .../scheduler/capacity/LeafQueue.java | 40 ++--- .../capacity/ManagedParentQueue.java | 8 +- .../scheduler/capacity/ParentQueue.java | 64 ++++---- .../capacity/QueueManagementChange.java | 2 +- .../QueueManagementDynamicEditPolicy.java | 4 +- .../scheduler/capacity/UsersManager.java | 2 +- .../WorkflowPriorityMappingsManager.java | 4 +- .../preemption/PreemptionManager.java | 6 +- ...uaranteedOrZeroCapacityOverTimePolicy.java | 38 ++--- .../common/fica/FiCaSchedulerApp.java | 7 +- .../security/QueueACLsManager.java | 16 +- .../TestReservationSystemWithRMHA.java | 2 +- ...CapacityPreemptionPolicyMockFramework.java | 17 +- ...tProportionalCapacityPreemptionPolicy.java | 12 +- ...PreemptionPolicyForReservedContainers.java | 20 +-- ...nalCapacityPreemptionPolicyIntraQueue.java | 8 +- ...acityPreemptionPolicyPreemptToBalance.java | 34 ++-- .../TestAppNameMappingPlacementRule.java | 11 ++ .../TestUserGroupMappingPlacementRule.java | 27 +++- .../ReservationSystemTestUtil.java | 17 +- .../reservation/TestReservationSystem.java | 8 +- .../scheduler/TestAbstractYarnScheduler.java | 8 +- .../TestAbsoluteResourceConfiguration.java | 8 +- .../capacity/TestApplicationLimits.java | 14 +- .../TestApplicationLimitsByPartition.java | 2 +- .../scheduler/capacity/TestCSQueueStore.java | 13 ++ .../capacity/TestCapacityScheduler.java | 25 +-- ...CapacitySchedulerAutoCreatedQueueBase.java | 12 +- ...estCapacitySchedulerAutoQueueCreation.java | 2 +- .../TestCapacitySchedulerLazyPreemption.java | 10 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 2 +- .../capacity/TestCapacitySchedulerPerf.java | 2 +- ...tCapacitySchedulerQueueMappingFactory.java | 4 +- ...stCapacitySchedulerSurgicalPreemption.java | 4 +- .../capacity/TestChildQueueOrder.java | 6 +- .../scheduler/capacity/TestLeafQueue.java | 20 +-- .../scheduler/capacity/TestParentQueue.java | 44 ++--- .../scheduler/capacity/TestQueueParsing.java | 2 +- .../scheduler/capacity/TestQueueState.java | 4 +- .../scheduler/capacity/TestReservations.java | 8 +- ...riorityUtilizationQueueOrderingPolicy.java | 4 +- .../TestRMWebServicesForCSWithPartitions.java | 7 +- .../TestRMWebServicesSchedulerActivities.java | 12 +- 67 files changed, 880 insertions(+), 521 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index d6f7bedda8f..9b2239043fc 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -380,6 +380,6 @@ public String getRealQueueName(String queue) throws YarnException { throw new YarnException("Can't find the queue by the given name: " + queue + "! Please check if queue " + queue + " is in the allocation file."); } - return getQueue(queue).getQueueName(); + return getQueue(queue).getQueuePath(); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java index ed50effee86..b16f2b4a596 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java @@ -83,7 +83,7 @@ public static void deductPreemptableResourcesBasedSelectedCandidates( String partition = schedulerNode.getPartition(); String queue = c.getQueueName(); - TempQueuePerPartition tq = context.getQueueByPartition(queue, + TempQueuePerPartition tq = context.getQueueByPartition(context.getScheduler().normalizeQueueName(queue), partition); Resource res = c.getReservedResource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java index 7c57f9c2cae..63ff2bbac32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java @@ -607,8 +607,8 @@ public boolean skipContainerBasedOnIntraQueuePolicy(FiCaSchedulerApp app, // might be due to high priority apps running in same user. String partition = context.getScheduler() .getSchedulerNode(c.getAllocatedNode()).getPartition(); - TempQueuePerPartition tq = context.getQueueByPartition(app.getQueueName(), - partition); + String queueName = context.getScheduler().getQueue(app.getQueueName()).getQueuePath(); + TempQueuePerPartition tq = context.getQueueByPartition(queueName, partition); TempUserPerPartition tmpUser = tq.getUsersPerPartition().get(app.getUser()); // Given user is not present, skip the check. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 352ea3dbaa6..0e3fbc6d798 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -567,7 +567,7 @@ private TempQueuePerPartition cloneQueues(CSQueue curQueue, // Acquire a read lock from Parent/LeafQueue. readLock.lock(); try { - String queueName = curQueue.getQueueName(); + String queueName = curQueue.getQueuePath(); QueueCapacities qc = curQueue.getQueueCapacities(); float absCap = qc.getAbsoluteCapacity(partitionToLookAt); float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java index a4bc53d0dcf..b2299686c3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils.extractQueuePath; import static org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils.getPlacementContext; import static org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils.isStaticQueueMapping; import static org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils.validateAndGetAutoCreatedQueueMapping; @@ -50,6 +49,7 @@ private boolean overrideWithQueueMappings = false; private List mappings = null; + protected CapacitySchedulerQueueManager queueManager; public AppNameMappingPlacementRule() { this(false, null); @@ -82,52 +82,56 @@ public boolean initialize(ResourceScheduler scheduler) // Get new user mappings List newMappings = new ArrayList<>(); - CapacitySchedulerQueueManager queueManager = - schedulerContext.getCapacitySchedulerQueueManager(); + queueManager = schedulerContext.getCapacitySchedulerQueueManager(); // check if mappings refer to valid queues + //TODO: Parts of this logic is very similar to UserGroupMappingPlacementRule + // it should be extracted for (QueueMappingEntity mapping : queueMappings) { + //TODO: Instead of the queue path we should just use mapping here + QueuePath queuePath = mapping.getQueuePath(); - QueuePath queuePath = extractQueuePath(mapping.getQueue()); if (isStaticQueueMapping(mapping)) { - //Try getting queue by its leaf queue name - // without splitting into parent/leaf queues - CSQueue queue = queueManager.getQueue(mapping.getQueue()); + //at this point mapping.getQueueName() return only the queue name, since + //the config parsing have been changed, this way QueueMapping behaves consistent + + //TODO: Mapping has all the relevant information, we really shouldn't use + // the QueuePath objects + CSQueue queue = queueManager.getQueue(mapping.getFullPath()); if (ifQueueDoesNotExist(queue)) { - //Try getting the queue by extracting leaf and parent queue names - //Assuming its a potential auto created leaf queue - queue = queueManager.getQueue(queuePath.getLeafQueue()); - - if (ifQueueDoesNotExist(queue)) { - //if leaf queue does not exist, - // this could be a potential auto created leaf queue - //validate if parent queue is specified, - // then it should exist and - // be an instance of AutoCreateEnabledParentQueue - QueueMappingEntity newMapping = - validateAndGetAutoCreatedQueueMapping(queueManager, mapping, - queuePath); - if (newMapping == null) { - throw new IOException( - "mapping contains invalid or non-leaf queue " + mapping - .getQueue()); - } - newMappings.add(newMapping); - } else{ - QueueMappingEntity newMapping = validateAndGetQueueMapping( - queueManager, queue, mapping, queuePath); - newMappings.add(newMapping); + //Try getting queue by its full path name, if it exists it is a static + //leaf queue indeed, without any auto creation magic + + if (queueManager.isAmbiguous(queuePath.getFullPath())) { + throw new IOException( + "mapping contains ambiguous leaf queue reference " + mapping + .getFullPath()); } - } else{ + + //if leaf queue does not exist, + // this could be a potential auto created leaf queue + //validate if parent queue is specified, + // then it should exist and + // be an instance of AutoCreateEnabledParentQueue + QueueMappingEntity newMapping = + validateAndGetAutoCreatedQueueMapping(queueManager, mapping, + queuePath); + if (newMapping == null) { + throw new IOException( + "mapping contains invalid or non-leaf queue " + mapping + .getQueue()); + } + newMappings.add(newMapping); + } else { // if queue exists, validate // if its an instance of leaf queue // if its an instance of auto created leaf queue, // then extract parent queue name and update queue mapping QueueMappingEntity newMapping = validateAndGetQueueMapping( - queueManager, queue, mapping, queuePath); + queueManager, queue, mapping, queuePath); newMappings.add(newMapping); } - } else{ + } else { //If it is a dynamic queue mapping, // we can safely assume leaf queue name does not have '.' in it // validate @@ -135,7 +139,7 @@ public boolean initialize(ResourceScheduler scheduler) // parent queue exists and an instance of AutoCreateEnabledParentQueue // QueueMappingEntity newMapping = validateAndGetAutoCreatedQueueMapping( - queueManager, mapping, queuePath); + queueManager, mapping, queuePath); if (newMapping != null) { newMappings.add(newMapping); } else{ @@ -163,13 +167,13 @@ private ApplicationPlacementContext getAppPlacementContext(String user, for (QueueMappingEntity mapping : mappings) { if (mapping.getSource().equals(CURRENT_APP_MAPPING)) { if (mapping.getQueue().equals(CURRENT_APP_MAPPING)) { - return getPlacementContext(mapping, applicationName); + return getPlacementContext(mapping, applicationName, queueManager); } else { - return getPlacementContext(mapping); + return getPlacementContext(mapping, queueManager); } } if (mapping.getSource().equals(applicationName)) { - return getPlacementContext(mapping); + return getPlacementContext(mapping, queueManager); } } return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java index bd0e0e296cd..9903ec3549b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement; import org.apache.hadoop.classification.InterfaceAudience.Private; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; /** * Queue Mapping class to hold the queue mapping information. @@ -65,6 +66,12 @@ public QueueMappingBuilder parentQueue(String mappingParentQueue) { return this; } + public QueueMappingBuilder queuePath(QueuePath path) { + this.queue = path.getLeafQueue(); + this.parentQueue = path.getParentQueue(); + return this; + } + public QueueMapping build() { return new QueueMapping(this); } @@ -124,6 +131,16 @@ public String getSource() { return source; } + public String getFullPath() { + return (parentQueue != null ? parentQueue + DOT + queue : queue); + } + + public QueuePath getQueuePath() { + //TODO: This is to make sure the parsing is the same everywhere, but the whole, + //parsing part should be moved to QueuePathConstructor + return QueuePlacementRuleUtils.extractQueuePath(getFullPath()); + } + @Override public int hashCode() { final int prime = 31; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java index c8cd299d98b..55947f14aee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java @@ -18,10 +18,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; + +//TODO: This class should be entirely replaced by QueueMapping public class QueueMappingEntity { private String source; private String queue; private String parentQueue; + private String fullPath; public final static String DELIMITER = ":"; @@ -29,11 +33,20 @@ public QueueMappingEntity(String source, String queue) { this.source = source; this.queue = queue; this.parentQueue = null; + this.fullPath = queue; } public QueueMappingEntity(String source, String queue, String parentQueue) { this.source = source; this.queue = queue; this.parentQueue = parentQueue; + this.fullPath = parentQueue + DOT + queue; + } + + public QueueMappingEntity(String source, QueuePath path) { + this.source = source; + this.queue = path.getLeafQueue(); + this.parentQueue = path.getParentQueue(); + this.fullPath = parentQueue + DOT + queue; } public String getQueue() { @@ -44,10 +57,24 @@ public String getParentQueue() { return parentQueue; } + public String getFullPath() { + return fullPath; + } + public String getSource() { return source; } + public boolean hasParentQueue() { + return parentQueue != null; + } + + public QueuePath getQueuePath() { + //TODO: This is to make sure the parsing is the same everywhere, but the whole, + //parsing part should be moved to QueuePathConstructor + return QueuePlacementRuleUtils.extractQueuePath(getFullPath()); + } + @Override public int hashCode() { return super.hashCode(); @@ -66,7 +93,7 @@ public boolean equals(Object obj) { public String toString() { return source + DELIMITER + (parentQueue != null ? - parentQueue + "." + queue : + parentQueue + DOT + queue : queue); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePath.java index 269353158a5..f11a296720c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePath.java @@ -24,14 +24,17 @@ private String parentQueue; private String leafQueue; + private String fullPath; public QueuePath(final String leafQueue) { this.leafQueue = leafQueue; + this.fullPath = leafQueue; } public QueuePath(final String parentQueue, final String leafQueue) { this.parentQueue = parentQueue; this.leafQueue = leafQueue; + this.fullPath = parentQueue + DOT + leafQueue; } public String getParentQueue() { @@ -46,8 +49,12 @@ public boolean hasParentQueue() { return parentQueue != null; } + public String getFullPath() { + return fullPath; + } + @Override public String toString() { - return parentQueue + DOT + leafQueue; + return fullPath; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java index 9463603f81d..627a031af2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java @@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; + import java.io.IOException; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; @@ -53,13 +55,13 @@ public static void validateQueueMappingUnderParentQueue( + "] and invalid parent queue which " + "does not have auto creation of leaf queues enabled [" + parentQueueName + "]"); - } else if (!parentQueue.getQueueName().equals(parentQueueName)) { + } else if (!parentQueue.getQueueShortName().equals(parentQueueName)) { throw new IOException( "mapping contains invalid or non-leaf queue [" + leafQueueName + "] and invalid parent queue " + "which does not match existing leaf queue's parent : [" + parentQueueName + "] does not match [ " + parentQueue - .getQueueName() + "]"); + .getQueueShortName() + "]"); } } @@ -71,9 +73,10 @@ public static QueueMappingEntity validateAndGetAutoCreatedQueueMapping( // then it should exist and be an instance of ManagedParentQueue validateQueueMappingUnderParentQueue(queueManager.getQueue( queuePath.getParentQueue()), queuePath.getParentQueue(), - queuePath.getLeafQueue()); + queuePath.getFullPath()); + //TODO: This is the same entity we just received no need for new one return new QueueMappingEntity(mapping.getSource(), - queuePath.getLeafQueue(), queuePath.getParentQueue()); + queuePath.getFullPath(), queuePath.getParentQueue()); } return null; @@ -84,7 +87,7 @@ public static QueueMappingEntity validateAndGetQueueMapping( QueueMappingEntity mapping, QueuePath queuePath) throws IOException { if (!(queue instanceof LeafQueue)) { throw new IOException( - "mapping contains invalid or non-leaf queue : " + mapping.getQueue()); + "mapping contains invalid or non-leaf queue : " + mapping.getFullPath()); } if (queue instanceof AutoCreatedLeafQueue && queue @@ -94,7 +97,7 @@ public static QueueMappingEntity validateAndGetQueueMapping( queueManager, mapping, queuePath); if (newMapping == null) { throw new IOException( - "mapping contains invalid or non-leaf queue " + mapping.getQueue()); + "mapping contains invalid or non-leaf queue " + mapping.getFullPath()); } return newMapping; } @@ -107,8 +110,8 @@ public static boolean isStaticQueueMapping(QueueMappingEntity mapping) { && !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING); } - public static QueuePath extractQueuePath(String queueName) - throws IOException { + //TODO: This really should be a constructor of QueuePath + public static QueuePath extractQueuePath(String queueName) { int parentQueueNameEndIndex = queueName.lastIndexOf(DOT); if (parentQueueNameEndIndex > -1) { @@ -122,13 +125,22 @@ public static QueuePath extractQueuePath(String queueName) return new QueuePath(queueName); } + //TODO: find a better way to pass queueManager public static ApplicationPlacementContext getPlacementContext( - QueueMappingEntity mapping) { - return getPlacementContext(mapping, mapping.getQueue()); + QueueMappingEntity mapping, CapacitySchedulerQueueManager queueManager) throws IOException { + return getPlacementContext(mapping, mapping.getQueue(), queueManager); } + //TODO: find a better way to pass queueManager public static ApplicationPlacementContext getPlacementContext( - QueueMappingEntity mapping, String leafQueueName) { + QueueMappingEntity mapping, String leafQueueName, + CapacitySchedulerQueueManager queueManager) throws IOException { + + //leafQueue name no longer identifies a queue uniquely we need to check for ambiguity + if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) { + throw new IOException("mapping contains ambiguous leaf queue reference " + leafQueueName); + } + if (!org.apache.commons.lang3.StringUtils.isEmpty(mapping.getParentQueue())) { return new ApplicationPlacementContext(leafQueueName, mapping.getParentQueue()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 6dce9c76c91..c285661946d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -95,35 +95,26 @@ private String getSecondaryGroup(String user) throws IOException { private ApplicationPlacementContext getPlacementForUser(String user) throws IOException { for (QueueMapping mapping : mappings) { - if (mapping.getType() == MappingType.USER) { + if (mapping.getType().equals(MappingType.USER)) { if (mapping.getSource().equals(CURRENT_USER_MAPPING)) { if (mapping.getParentQueue() != null - && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) - && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { - if (this.queueManager - .getQueue(groups.getGroups(user).get(0)) != null) { - QueueMapping queueMapping = - QueueMappingBuilder.create() - .type(mapping.getType()) - .source(mapping.getSource()).queue(user) - .parentQueue(groups.getGroups(user).get(0)) - .build(); - validateQueueMapping(queueMapping); - return getPlacementContext(queueMapping, user); - } else { - return null; - } - } else if (mapping.getParentQueue() != null - && mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING) - && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { - String secondaryGroup = getSecondaryGroup(user); - if (secondaryGroup != null) { + && mapping.getQueue().equals(CURRENT_USER_MAPPING) + && (mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) + || mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING) + )) { + //TODO: If we'd use contains instead of equals, and we'd just replace + // the tokens, and would set those as parentQueueName, we could enable + // deeper structures than groupName.userName + String parentName = CapacitySchedulerConfiguration.ROOT + "." + + (mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) ? + groups.getGroups(user).get(0) : getSecondaryGroup(user)); + if (this.queueManager.getQueueByFullName(parentName) != null) { QueueMapping queueMapping = QueueMappingBuilder.create() .type(mapping.getType()) .source(mapping.getSource()) .queue(user) - .parentQueue(secondaryGroup) + .parentQueue(parentName) .build(); validateQueueMapping(queueMapping); return getPlacementContext(queueMapping, user); @@ -134,9 +125,9 @@ private ApplicationPlacementContext getPlacementForUser(String user) return getPlacementContext(mapping, user); } else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { if (this.queueManager - .getQueue(groups.getGroups(user).get(0)) != null) { + .getQueue(CapacitySchedulerConfiguration.ROOT + "." + groups.getGroups(user).get(0)) != null) { return getPlacementContext(mapping, - groups.getGroups(user).get(0)); + CapacitySchedulerConfiguration.ROOT + "." + groups.getGroups(user).get(0)); } else { return null; } @@ -166,7 +157,7 @@ private ApplicationPlacementContext getPlacementForUser(String user) } } } - if (mapping.getType() == MappingType.GROUP) { + if (mapping.getType().equals(MappingType.GROUP)) { for (String userGroups : groups.getGroups(user)) { if (userGroups.equals(mapping.getSource())) { if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { @@ -211,12 +202,18 @@ public ApplicationPlacementContext getPlacementForApp( } private ApplicationPlacementContext getPlacementContext( - QueueMapping mapping) { + QueueMapping mapping) throws IOException { return getPlacementContext(mapping, mapping.getQueue()); } private ApplicationPlacementContext getPlacementContext(QueueMapping mapping, - String leafQueueName) { + String leafQueueName) throws IOException { + + //leafQueue name no longer identifies a queue uniquely we need to check for ambiguity + if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) { + throw new IOException("mapping contains ambiguous leaf queue reference " + leafQueueName); + } + if (!StringUtils.isEmpty(mapping.getParentQueue())) { return new ApplicationPlacementContext(leafQueueName, mapping.getParentQueue()); @@ -250,38 +247,39 @@ public boolean initialize(ResourceScheduler scheduler) // check if mappings refer to valid queues for (QueueMapping mapping : queueMappings) { + //at this point mapping.getQueueName() return only the queue name, since + //the config parsing have been changed, this way QueueMapping behaves consistent - QueuePath queuePath = QueuePlacementRuleUtils - .extractQueuePath(mapping.getQueue()); + //TODO: Mapping has all the relevant information, we really shouldn't use + // the QueuePath objects + QueuePath queuePath = mapping.getQueuePath(); if (isStaticQueueMapping(mapping)) { - //Try getting queue by its leaf queue name - // without splitting into parent/leaf queues - CSQueue queue = queueManager.getQueue(mapping.getQueue()); + //Try getting queue by its full path name, if it exists it is a static + //leaf queue indeed, without any auto creation magic + CSQueue queue = queueManager.getQueue(mapping.getFullPath()); if (ifQueueDoesNotExist(queue)) { - //Try getting the queue by extracting leaf and parent queue names - //Assuming its a potential auto created leaf queue - queue = queueManager.getQueue(queuePath.getLeafQueue()); - - if (ifQueueDoesNotExist(queue)) { - //if leaf queue does not exist, - // this could be a potential auto created leaf queue - //validate if parent queue is specified, - // then it should exist and - // be an instance of AutoCreateEnabledParentQueue - QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping( - queueManager, mapping, queuePath); - if (newMapping == null) { - throw new IOException( - "mapping contains invalid or non-leaf queue " + mapping - .getQueue()); - } - newMappings.add(newMapping); - } else{ - QueueMapping newMapping = validateAndGetQueueMapping(queueManager, - queue, mapping, queuePath); - newMappings.add(newMapping); + //We might not be able to find the queue, because the reference was ambiguous + //this should only happen if the queue was referenced by leaf name only + if (queueManager.isAmbiguous(mapping.getFullPath())) { + throw new IOException( + "mapping contains ambiguous leaf queue reference " + mapping + .getFullPath()); } - } else{ + + //if leaf queue does not exist, + // this could be a potential auto created leaf queue + //validate if parent queue is specified, + // then it should exist and + // be an instance of AutoCreateEnabledParentQueue + QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping( + queueManager, mapping, queuePath); + if (newMapping == null) { + throw new IOException( + "mapping contains invalid or non-leaf queue " + mapping + .getQueue()); + } + newMappings.add(newMapping); + } else { // if queue exists, validate // if its an instance of leaf queue // if its an instance of auto created leaf queue, @@ -323,7 +321,7 @@ private static QueueMapping validateAndGetQueueMapping( QueueMapping mapping, QueuePath queuePath) throws IOException { if (!(queue instanceof LeafQueue)) { throw new IOException( - "mapping contains invalid or non-leaf queue : " + mapping.getQueue()); + "mapping contains invalid or non-leaf queue : " + mapping.getFullPath()); } if (queue instanceof AutoCreatedLeafQueue && queue @@ -333,7 +331,7 @@ private static QueueMapping validateAndGetQueueMapping( queueManager, mapping, queuePath); if (newMapping == null) { throw new IOException( - "mapping contains invalid or non-leaf queue " + mapping.getQueue()); + "mapping contains invalid or non-leaf queue " + mapping.getFullPath()); } return newMapping; } @@ -344,6 +342,7 @@ private static boolean ifQueueDoesNotExist(CSQueue queue) { return queue == null; } + //TODO: Extract all neccessary information from mapping private static QueueMapping validateAndGetAutoCreatedQueueMapping( CapacitySchedulerQueueManager queueManager, QueueMapping mapping, QueuePath queuePath) throws IOException { @@ -351,6 +350,7 @@ private static QueueMapping validateAndGetAutoCreatedQueueMapping( && (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING) || queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) { // dynamic parent queue + //TODO: This is technically the same mapping we've just received return QueueMappingBuilder.create() .type(mapping.getType()) .source(mapping.getSource()) @@ -363,6 +363,7 @@ private static QueueMapping validateAndGetAutoCreatedQueueMapping( QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue( queueManager.getQueue(queuePath.getParentQueue()), queuePath.getParentQueue(), queuePath.getLeafQueue()); + //TODO: This is technically the same mapping we've just received return QueueMappingBuilder.create() .type(mapping.getType()) .source(mapping.getSource()) @@ -386,22 +387,31 @@ private static boolean isStaticQueueMapping(QueueMapping mapping) { private void validateQueueMapping(QueueMapping queueMapping) throws IOException { String parentQueueName = queueMapping.getParentQueue(); - String leafQueueName = queueMapping.getQueue(); - CSQueue parentQueue = queueManager.getQueue(parentQueueName); - CSQueue leafQueue = queueManager.getQueue(leafQueueName); + String leafQueueFullName = queueMapping.getFullPath(); + CSQueue parentQueue = queueManager.getQueueByFullName(parentQueueName); + CSQueue leafQueue = queueManager.getQueue(leafQueueFullName); if (leafQueue == null || (!(leafQueue instanceof LeafQueue))) { - throw new IOException("mapping contains invalid or non-leaf queue : " - + leafQueueName); + //this might be confusing, but a mapping is not guaranteed to provide the + //parent queue's name, which can result in ambiguous queue references + //if no parent queueName is provided mapping.getFullPath() is the same + //as mapping.getQueue() + if (leafQueue == null && queueManager.isAmbiguous(leafQueueFullName)) { + throw new IOException("mapping contains ambiguous leaf queue name: " + + leafQueueFullName); + } else { + throw new IOException("mapping contains invalid or non-leaf queue : " + + leafQueueFullName); + } } else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) { throw new IOException( "mapping contains invalid parent queue [" + parentQueueName + "]"); - } else if (!parentQueue.getQueueName() - .equals(leafQueue.getParent().getQueueName())) { + } else if (!parentQueue.getQueuePath() + .equals(leafQueue.getParent().getQueuePath())) { throw new IOException("mapping contains invalid parent queue " + "which does not match existing leaf queue's parent : [" - + parentQueue.getQueueName() + "] does not match [ " - + leafQueue.getParent().getQueueName() + "]"); + + parentQueue.getQueuePath() + "] does not match [ " + + leafQueue.getParent().getQueuePath() + "]"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java index be0a7088fcb..c1569816cf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityReservationSystem.java @@ -99,6 +99,11 @@ protected Resource getPlanQueueCapacity(String planQueueName) { planQueue.getAbsoluteCapacity(), minAllocation); } + @Override + public Plan getPlan(String planName) { + return super.getPlan(capScheduler.normalizeQueueName(planName)); + } + @Override protected ReservationSchedulerConfiguration getReservationSchedulerConfiguration() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index d7426833ac8..4f0678a2794 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -34,6 +34,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -553,9 +554,15 @@ public void recoverContainersOnNode(List containerReports, } } + Queue queue = schedulerApp.getQueue(); + //To make sure we don't face ambiguity, CS queues should be referenced + //by their full queue names + String queueName = queue instanceof CSQueue ? + ((CSQueue)queue).getQueuePath() : queue.getQueueName(); + // create container RMContainer rmContainer = recoverAndCreateContainer(container, nm, - schedulerApp.getQueue().getQueueName()); + queueName); // recover RMContainer rmContainer.handle( @@ -566,8 +573,8 @@ public void recoverContainersOnNode(List containerReports, schedulerNode.recoverContainer(rmContainer); // recover queue: update headroom etc. - Queue queue = schedulerAttempt.getQueue(); - queue.recoverContainer(getClusterResource(), schedulerAttempt, + Queue queueToRecover = schedulerAttempt.getQueue(); + queueToRecover.recoverContainer(getClusterResource(), schedulerAttempt, rmContainer); // recover scheduler attempt diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 8149ab470ff..1a1f40145f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -36,6 +36,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.lang3.time.FastDateFormat; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -588,7 +590,13 @@ public RMContainer reserve(SchedulerNode node, if (rmContainer.getState() == RMContainerState.NEW) { attemptResourceUsage.incReserved(node.getPartition(), container.getResource()); - ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName()); + + ResourceScheduler scheduler = this.rmContext.getScheduler(); + String qn = this.getQueueName(); + if (scheduler instanceof CapacityScheduler) { + qn = ((CapacityScheduler)scheduler).normalizeQueueName(qn); + } + ((RMContainerImpl) rmContainer).setQueueName(qn); // Reset the re-reservation count resetReReservations(schedulerKey); @@ -1182,7 +1190,8 @@ public void move(Queue newQueue) { try { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); - String newQueueName = newQueue.getQueueName(); + String newQueueName = newQueue instanceof CSQueue ? + ((CSQueue) newQueue).getQueuePath() : newQueue.getQueueName(); String user = getUser(); for (RMContainer liveContainer : liveContainers.values()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java index 77ed2182f19..cc02ff688cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java @@ -22,6 +22,7 @@ import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.commons.collections.CollectionUtils; @@ -385,6 +386,15 @@ void addSchedulingActivityForNode(NodeId nodeId, String parentName, String diagnostic, ActivityLevel level, Long allocationRequestId) { if (shouldRecordThisNode(nodeId)) { NodeAllocation nodeAllocation = getCurrentNodeAllocation(nodeId); + + ResourceScheduler scheduler = this.rmContext.getScheduler(); + //Sorry about this :( Making sure CS short queue references are normalized + if (scheduler instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler)this.rmContext.getScheduler(); + parentName = cs.normalizeQueueName(parentName); + childName = cs.normalizeQueueName(childName); + } + nodeAllocation.addAllocationActivity(parentName, childName, priority, state, diagnostic, level, nodeId, allocationRequestId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 9e5bdb0bc0b..bed2aa5b06d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -93,7 +93,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement) // this might be revised later setMaxCapacity(nodeLabel, entitlement.getMaxCapacity()); LOG.debug("successfully changed to {} for queue {}", capacity, this - .getQueueName()); + .getQueuePath()); //update queue used capacity etc CSQueueUtils.updateQueueStatistics(resourceCalculator, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 4345cbd5a3f..280e7238415 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -110,6 +110,8 @@ // Track resource usage-by-label like used-resource/pending-resource, etc. volatile ResourceUsage queueUsage; + + private final boolean fullPathQueueNamingPolicy = false; // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. @@ -203,7 +205,7 @@ protected void setupConfigurableCapacities( public String getQueuePath() { return queuePath; } - + @Override public float getCapacity() { return queueCapacities.getCapacity(); @@ -253,8 +255,16 @@ public CSQueueMetrics getMetrics() { return metrics; } + @Override + public String getQueueShortName() { + return queueName; + } + @Override public String getQueueName() { + if (fullPathQueueNamingPolicy) { + return queuePath; + } return queueName; } @@ -292,11 +302,11 @@ void setMaxCapacity(float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueueName(), + CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueueName(), + CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -314,11 +324,11 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueueName(), + CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(nodeLabel), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueueName(), + CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -526,7 +536,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, queuePath, resourceTypes); LOG.debug("capacityConfigType is '{}' for queue {}", - capacityConfigType, getQueueName()); + capacityConfigType, getQueuePath()); if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { this.capacityConfigType = (!minResource.equals(Resources.none()) @@ -534,7 +544,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, ? CapacityConfigType.ABSOLUTE_RESOURCE : CapacityConfigType.PERCENTAGE; LOG.debug("capacityConfigType is updated as '{}' for queue {}", - capacityConfigType, getQueueName()); + capacityConfigType, getQueuePath()); } validateAbsoluteVsPercentageCapacityConfig(minResource); @@ -545,7 +555,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, resourceCalculator, clusterResource, minResource, maxResource)) { throw new IllegalArgumentException("Min resource configuration " + minResource + " is greater than its max value:" + maxResource - + " in queue:" + getQueueName()); + + " in queue:" + getQueuePath()); } // If parent's max resource is lesser to a specific child's max @@ -559,7 +569,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, maxResource, parentMaxRes)) { throw new IllegalArgumentException("Max resource configuration " + maxResource + " is greater than parents max value:" - + parentMaxRes + " in queue:" + getQueueName()); + + parentMaxRes + " in queue:" + getQueuePath()); } // If child's max resource is not set, but its parent max resource is @@ -572,7 +582,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, } LOG.debug("Updating absolute resource configuration for queue:{} as" - + " minResource={} and maxResource={}", getQueueName(), minResource, + + " minResource={} and maxResource={}", getQueuePath(), minResource, maxResource); queueResourceQuotas.setConfiguredMinResource(label, minResource); @@ -587,9 +597,9 @@ private void validateAbsoluteVsPercentageCapacityConfig( localType = CapacityConfigType.ABSOLUTE_RESOURCE; } - if (!queueName.equals("root") + if (!queuePath.equals("root") && !this.capacityConfigType.equals(localType)) { - throw new IllegalArgumentException("Queue '" + getQueueName() + throw new IllegalArgumentException("Queue '" + getQueuePath() + "' should use either percentage based capacity" + " configuration or absolute resource."); } @@ -651,8 +661,8 @@ private void initializeQueueState(QueueState previousState, } else if (configuredState == QueueState.RUNNING && parentState != QueueState.RUNNING) { throw new IllegalArgumentException( - "The parent queue:" + parent.getQueueName() - + " cannot be STOPPED as the child queue:" + queueName + "The parent queue:" + parent.getQueuePath() + + " cannot be STOPPED as the child queue:" + queuePath + " is in RUNNING state."); } else { updateQueueState(configuredState); @@ -1067,7 +1077,7 @@ boolean canAssignToThisQueue(Resource clusterResource, if (Resources.lessThan(resourceCalculator, clusterResource, newTotalWithoutReservedResource, currentLimitResource)) { if (LOG.isDebugEnabled()) { - LOG.debug("try to use reserved: " + getQueueName() + LOG.debug("try to use reserved: " + getQueuePath() + " usedResources: " + queueUsage.getUsed() + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved @@ -1081,7 +1091,7 @@ boolean canAssignToThisQueue(Resource clusterResource, // Can not assign to this queue if (LOG.isDebugEnabled()) { - LOG.debug("Failed to assign to queue: " + getQueueName() + LOG.debug("Failed to assign to queue: " + getQueuePath() + " nodePatrition: " + nodePartition + ", usedResources: " + queueUsage.getUsed(nodePartition) + ", clusterResources: " + clusterResource @@ -1092,7 +1102,7 @@ boolean canAssignToThisQueue(Resource clusterResource, return false; } if (LOG.isDebugEnabled()) { - LOG.debug("Check assign to queue: " + getQueueName() + LOG.debug("Check assign to queue: " + getQueuePath() + " nodePartition: " + nodePartition + ", usedResources: " + queueUsage.getUsed(nodePartition) + ", clusterResources: " + clusterResource @@ -1248,12 +1258,12 @@ public Priority getDefaultApplicationPriority() { } public Resource getTotalKillableResource(String partition) { - return csContext.getPreemptionManager().getKillableResource(queueName, + return csContext.getPreemptionManager().getKillableResource(getQueuePath(), partition); } public Iterator getKillableContainers(String partition) { - return csContext.getPreemptionManager().getKillableContainers(queueName, + return csContext.getPreemptionManager().getKillableContainers(getQueuePath(), partition); } @@ -1336,18 +1346,18 @@ public void activeQueue() throws YarnException { this.writeLock.lock(); try { if (getState() == QueueState.RUNNING) { - LOG.info("The specified queue:" + queueName + LOG.info("The specified queue:" + getQueuePath() + " is already in the RUNNING state."); } else if (getState() == QueueState.DRAINING) { throw new YarnException( - "The queue:" + queueName + " is in the Stopping process. " + "The queue:" + getQueuePath() + " is in the Stopping process. " + "Please wait for the queue getting fully STOPPED."); } else { CSQueue parent = getParent(); if (parent == null || parent.getState() == QueueState.RUNNING) { updateQueueState(QueueState.RUNNING); } else { - throw new YarnException("The parent Queue:" + parent.getQueueName() + throw new YarnException("The parent Queue:" + parent.getQueuePath() + " is not running. Please activate the parent queue first"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 7626c66d555..7bdc311aea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -106,7 +106,7 @@ public void removeChildQueue(CSQueue childQueue) CSQueue cs = qiter.next(); if (cs.equals(childQueue)) { qiter.remove(); - LOG.debug("Removed child queue: {}", cs.getQueueName()); + LOG.debug("Removed child queue: {}", cs.getQueuePath()); } } } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index b194ad804d8..257aac66a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -56,7 +56,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) ManagedParentQueue managedParentQueue = (ManagedParentQueue) parent; super.reinitialize(newlyParsedQueue, clusterResource, managedParentQueue - .getLeafQueueConfigs(newlyParsedQueue.getQueueName())); + .getLeafQueueConfigs(newlyParsedQueue.getQueueShortName())); //Reset capacities to 0 since reinitialize above // queueCapacities to initialize to configured capacity which might diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 9fb7e7d0cc3..70feae56244 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -73,11 +73,17 @@ public void setParent(CSQueue newParentQueue); /** - * Get the queue name. + * Get the queue's internal reference name. * @return the queue name */ public String getQueueName(); + /** + * Get the queue's legacy name + * @return the queue name + */ + public String getQueueShortName(); + /** * Get the full name of the queue, including the heirarchy. * @return the full name of the queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java new file mode 100644 index 00000000000..d5fdcdb9ac2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java @@ -0,0 +1,151 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class CSQueueStore { + private final Map fullNameQueues = new ConcurrentHashMap<>(); + private final Map shortNameToFullName = new HashMap<>(); + private final Map> ambiguousShortNames = new HashMap<>(); + + public Map getFullNameQueues() { + return ImmutableMap.copyOf(fullNameQueues); + } + + public Map getShortNameQueues() { + return ImmutableMap.copyOf( + shortNameToFullName.entrySet().stream().collect( + Collectors.toMap( + Map.Entry::getKey, + entry->fullNameQueues.get(entry.getValue()))) + ); + +// shortNameToFullName.entrySet().stream().forEach(e -> System.out.println("<>" + e)); +// return null; + } + + //we must synchronize here because we need to maintain multiple maps to be + //in sync, and concurrent hashMap does not help with that + private synchronized void addShortNameMapping(String shortName, String fullName) { + if (ambiguousShortNames.containsKey(shortName)) { + //if the short name is already ambiguous we just add it to the list + ambiguousShortNames.get(shortName).add(fullName); + } else if (shortNameToFullName.containsKey(shortName) + && !fullName.equals(shortNameToFullName.get(shortName))) { + + //the short name became ambiguous creating the ambiguous list + Set ambiguousQueues = new HashSet<>(); + ambiguousQueues.add(fullName); + ambiguousQueues.add(shortNameToFullName.get(shortName)); + ambiguousShortNames.put(shortName, ambiguousQueues); + + //removing the short name from queue name map, since there is no + //disambiguous mapping + shortNameToFullName.remove(shortName); + } else { + //there is no issue with the queue's short name we just add it to the map + shortNameToFullName.put(shortName, fullName); + } + } + + public void add(CSQueue queue) { + String fullName = queue.getQueuePath(); + String shortName = queue.getQueueShortName(); + + fullNameQueues.put(fullName, queue); +// if (queue instanceof LeafQueue) { + addShortNameMapping(shortName, fullName); +// } + } + + public synchronized void remove(CSQueue queue) { + //if no queue is specified, we can consider it already removed, also consistent + //with hashmap behaviour, so no new issues will be caused by it + if (queue == null) { + return; + } + + String fullName = queue.getQueuePath(); + String shortName = queue.getQueueShortName(); + + //removing from the full and short name maps as well + fullNameQueues.remove(fullName); + if (shortNameToFullName.containsKey(shortName)) { + shortNameToFullName.remove(shortName); + } + + if (ambiguousShortNames.containsKey(shortName) /*&& queue instanceof LeafQueue*/) { + //if the removed queued was an ambiguous one, we check if removing it + //resolves the issue. + Set conflictingQueues = ambiguousShortNames.get(shortName); + conflictingQueues.remove(fullName); + if (conflictingQueues.size() == 1) { + //if we have only 1 conflictingQueue, it is not a conflict, resolving + ambiguousShortNames.remove(shortName); + shortNameToFullName.put(shortName, conflictingQueues.iterator().next()); + } + } + + } + + + public synchronized void remove(String name) { + CSQueue queue = get(name); + if (queue != null) remove(queue); + } + + public int getQueueCount() { + return fullNameQueues.size(); + } + + public CSQueue getByFullName(String name) { + if (name == null) return null; + return fullNameQueues.getOrDefault(name, null); + } + + public CSQueue getByShortName(String name) { + if (name == null) return null; + String mappedName = shortNameToFullName.getOrDefault(name, null); + if (mappedName != null) { + return fullNameQueues.get(mappedName); + } + + return null; + } + + public boolean isAmbiguous(String shortName) { + if (shortName == null) return false; + return ambiguousShortNames.get(shortName) != null; + } + + public CSQueue get(String name) { + if (name == null) return null; + + CSQueue ret = getByFullName(name); + + if (ret == null) { + ret = getByShortName(name); + } + + return ret; + } + + public synchronized void clear() { + fullNameQueues.clear(); + shortNameToFullName.clear(); + ambiguousShortNames.clear(); + } + + public Collection getQueues() { + return ImmutableList.copyOf(fullNameQueues.values()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 174a699545a..1114d195504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -778,6 +778,15 @@ public CSQueue getQueue(String queueName) { return this.queueManager.getQueue(queueName); } + public String normalizeQueueName(String name) { + if (this.queueManager == null) return name; + return this.queueManager.normalizeQueueName(name); + } + + public boolean isAmbiguous(String queueName) { + return this.queueManager.isAmbiguous(queueName); + } + private void addApplicationOnRecovery(ApplicationId applicationId, String queueName, String user, Priority priority, ApplicationPlacementContext placementContext) { @@ -955,12 +964,12 @@ private void addApplication(ApplicationId applicationId, String queueName, // For a queue which exists already and // not auto-created above, then its parent queue should match // the parent queue specified in queue mapping - } else if (!queue.getParent().getQueueName().equals( + } else if (!queue.getParent().getQueueShortName().equals( placementContext.getParentQueue())) { String message = "Auto created Leaf queue " + placementContext.getQueue() + " " + "already exists under queue : " + queue - .getParent().getQueuePath() + .getParent().getQueueShortName() + ".But Queue mapping configuration " + CapacitySchedulerConfiguration.QUEUE_MAPPING + " has been " + "updated to a different parent queue : " @@ -1048,7 +1057,7 @@ private void addApplicationAttempt( queue.submitApplicationAttempt(attempt, application.getUser()); LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user " + application.getUser() + " in queue " - + queue.getQueueName()); + + queue.getQueuePath()); if (isAttemptRecovering) { LOG.debug("{} is recovering. Skipping notifying ATTEMPT_ADDED", applicationAttemptId); @@ -1077,7 +1086,7 @@ private void doneApplication(ApplicationId applicationId, CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue - .getQueueName()); + .getQueuePath()); } else{ queue.finishApplication(applicationId, application.getUser()); } @@ -1132,13 +1141,15 @@ private void doneApplicationAttempt( attempt.stop(rmAppAttemptFinalState); // Inform the queue - String queueName = attempt.getQueue().getQueueName(); - CSQueue queue = this.getQueue(queueName); - if (!(queue instanceof LeafQueue)) { + Queue queue = attempt.getQueue(); + CSQueue csQueue = queue instanceof CSQueue ? + (CSQueue) queue : this.getQueue(queue.getQueueName()); + if (!(csQueue instanceof LeafQueue)) { LOG.error( - "Cannot finish application " + "from non-leaf queue: " + queueName); + "Cannot finish application " + "from non-leaf queue: " + + csQueue.getQueuePath()); } else{ - queue.finishApplicationAttempt(attempt, queue.getQueueName()); + csQueue.finishApplicationAttempt(attempt, csQueue.getQueuePath()); } } finally { writeLock.unlock(); @@ -1516,7 +1527,7 @@ private CSAssignment allocateContainerOnSingleNode( LOG.error("Trying to schedule on a removed node, please double check, " + "nodeId=" + node.getNodeID()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.INIT_CHECK_SINGLE_NODE_REMOVED); ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, node); @@ -1545,7 +1556,7 @@ private CSAssignment allocateContainerOnSingleNode( LOG.debug("This node " + node.getNodeID() + " doesn't have sufficient " + "available or preemptible resource for minimum allocation"); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant. INIT_CHECK_SINGLE_NODE_RESOURCE_INSUFFICIENT); ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -1594,14 +1605,14 @@ private void allocateFromReservedContainer(FiCaSchedulerNode node, schedulerHealth.updateSchedulerFulfilledReservationCounts(1); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), + queue.getParent().getQueuePath(), queue.getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.ALLOCATED_FROM_RESERVED); } else if (assignment.getAssignmentInformation().getNumReservations() > 0) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), + queue.getParent().getQueuePath(), queue.getQueuePath(), ActivityState.RE_RESERVED, ActivityDiagnosticConstant.EMPTY); ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.RESERVED); @@ -1693,7 +1704,7 @@ private CSAssignment allocateContainersOnMultiNodes( LOG.debug("This partition '{}' doesn't have available or " + "killable resource", candidates.getPartition()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, null, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant. INIT_CHECK_PARTITION_RESOURCE_INSUFFICIENT); ActivitiesLogger.NODE @@ -1907,10 +1918,10 @@ public void handle(SchedulerEvent event) { .validateAndApplyQueueManagementChanges(queueManagementChanges); } catch (SchedulerDynamicEditException sde) { LOG.error("Queue Management Change event cannot be applied for " - + "parent queue : " + parentQueue.getQueueName(), sde); + + "parent queue : " + parentQueue.getQueuePath(), sde); } catch (IOException ioe) { LOG.error("Queue Management Change event cannot be applied for " - + "parent queue : " + parentQueue.getQueueName(), ioe); + + "parent queue : " + parentQueue.getQueuePath(), ioe); } } break; @@ -2184,7 +2195,7 @@ public void markContainerForKillable( // notify PreemptionManager // Get the application for the finished container if (null != application) { - String leafQueueName = application.getCSLeafQueue().getQueueName(); + String leafQueueName = application.getCSLeafQueue().getQueuePath(); getPreemptionManager().addKillableContainer( new KillableContainer(killableContainer, node.getPartition(), leafQueueName)); @@ -2213,7 +2224,7 @@ private void markContainerForNonKillable( // notify PreemptionManager // Get the application for the finished container if (null != application) { - String leafQueueName = application.getCSLeafQueue().getQueueName(); + String leafQueueName = application.getCSLeafQueue().getQueuePath(); getPreemptionManager().removeKillableContainer( new KillableContainer(nonKillableContainer, node.getPartition(), leafQueueName)); @@ -2286,7 +2297,7 @@ private String resolveReservationQueueName(String queueName, message)); return null; } - if (!queue.getParent().getQueueName().equals(queueName)) { + if (!queue.getParent().getQueuePath().equals(queueName)) { String message = "Application: " + applicationId + " submitted to a reservation " + resQName + " which does not belong to the specified queue: " @@ -2371,7 +2382,7 @@ public void addQueue(Queue queue) AbstractManagedParentQueue parent = (AbstractManagedParentQueue) newQueue.getParent(); - String queuename = newQueue.getQueueName(); + String queuename = newQueue.getQueuePath(); parent.addChildQueue(newQueue); this.queueManager.addQueue(queuename, newQueue); @@ -2485,7 +2496,9 @@ public void preValidateMoveApplication(ApplicationId appId, if (application == null) { throw new YarnException("App to be moved " + appId + " not found."); } - String sourceQueueName = application.getQueue().getQueueName(); + Queue queue = application.getQueue(); + String sourceQueueName = queue instanceof CSQueue ? + ((CSQueue) queue).getQueuePath() : queue.getQueueName(); this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(newQueue); LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); @@ -2534,7 +2547,7 @@ private void checkQueuePartition(FiCaSchedulerApp app, LeafQueue dest) } if (nonAccessiblelabels.size() > 0) { throw new YarnException( - "Specified queue=" + dest.getQueueName() + " can't satisfy following " + "Specified queue=" + dest.getQueuePath() + " can't satisfy following " + "apps label expressions =" + nonAccessiblelabels + " accessible node labels =" + targetqueuelabels); } @@ -2611,14 +2624,14 @@ public Priority checkAndGetApplicationPriority( // user's default priority will get precedence over queue default. // for updateApplicationPriority call flow, this check is done in // CientRMService itself. - appPriority = this.appPriorityACLManager.getDefaultPriority(queueName, + appPriority = this.appPriorityACLManager.getDefaultPriority(this.normalizeQueueName(queueName), user); // Get the default priority for the Queue. If Queue is non-existent, // then // use default priority. Do it only if user doesn't have any default. if (null == appPriority) { - appPriority = this.queueManager.getDefaultPriorityForQueue(queueName); + appPriority = this.queueManager.getDefaultPriorityForQueue(this.normalizeQueueName(queueName)); } LOG.info( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 3957c5fcab1..c67b2bea113 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -112,15 +112,14 @@ public static void validateVCores(Configuration conf) { * @param queues existing queues * @param newQueues new queues */ - public static void validateQueueHierarchy(Map queues, - Map newQueues, CapacitySchedulerConfiguration newConf) - throws IOException { + public static void validateQueueHierarchy(CSQueueStore queues, + CSQueueStore newQueues, CapacitySchedulerConfiguration newConf) + throws IOException { // check that all static queues are included in the newQueues list - for (Map.Entry e : queues.entrySet()) { - if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue() - .getClass()))) { - String queueName = e.getKey(); - CSQueue oldQueue = e.getValue(); + for (CSQueue oldQueue : queues.getQueues()) { + if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(oldQueue + .getClass()))) { + String queueName = oldQueue.getQueuePath(); CSQueue newQueue = newQueues.get(queueName); String configPrefix = newConf.getQueuePrefix( oldQueue.getQueuePath()); @@ -137,55 +136,55 @@ public static void validateQueueHierarchy(Map queues, if (null == newQueue) { // old queue doesn't exist in the new XML if (oldQueue.getState() == QueueState.STOPPED || - newQueueState == QueueState.STOPPED) { + newQueueState == QueueState.STOPPED) { LOG.info("Deleting Queue " + queueName + ", as it is not" - + " present in the modified capacity configuration xml"); + + " present in the modified capacity configuration xml"); } else{ throw new IOException(oldQueue.getQueuePath() + " cannot be" - + " deleted from the capacity scheduler configuration, " - + "as the queue is not yet in stopped state. " - + "Current State : " + oldQueue.getState()); + + " deleted from the capacity scheduler configuration, as the" + + " queue is not yet in stopped state. Current State : " + + oldQueue.getState()); } } else if (!oldQueue.getQueuePath().equals(newQueue.getQueuePath())) { //Queue's cannot be moved from one hierarchy to other throw new IOException( - queueName + " is moved from:" + oldQueue.getQueuePath() - + " to:" + newQueue.getQueuePath() - + " after refresh, which is not allowed."); + queueName + " is moved from:" + oldQueue.getQueuePath() + " to:" + + newQueue.getQueuePath() + + " after refresh, which is not allowed."); } else if (oldQueue instanceof ParentQueue - && !(oldQueue instanceof ManagedParentQueue) - && newQueue instanceof ManagedParentQueue) { + && !(oldQueue instanceof ManagedParentQueue) + && newQueue instanceof ManagedParentQueue) { throw new IOException( - "Can not convert parent queue: " + oldQueue.getQueuePath() - + " to auto create enabled parent queue since " - + "it could have other pre-configured queues which is" - + " not supported"); + "Can not convert parent queue: " + oldQueue.getQueuePath() + + " to auto create enabled parent queue since " + + "it could have other pre-configured queues which is not " + + "supported"); } else if (oldQueue instanceof ManagedParentQueue - && !(newQueue instanceof ManagedParentQueue)) { + && !(newQueue instanceof ManagedParentQueue)) { throw new IOException( - "Cannot convert auto create enabled parent queue: " + oldQueue - .getQueuePath() + " to leaf queue. Please check " - + " parent queue's configuration " - + CapacitySchedulerConfiguration - .AUTO_CREATE_CHILD_QUEUE_ENABLED - + " is set to true"); + "Cannot convert auto create enabled parent queue: " + oldQueue + .getQueuePath() + " to leaf queue. Please check " + + " parent queue's configuration " + + CapacitySchedulerConfiguration + .AUTO_CREATE_CHILD_QUEUE_ENABLED + + " is set to true"); } else if (oldQueue instanceof LeafQueue && newQueue instanceof ParentQueue) { if (oldQueue.getState() == QueueState.STOPPED || newQueueState == QueueState.STOPPED) { LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath() - + " to parent queue."); + + " to parent queue."); } else{ throw new IOException( - "Can not convert the leaf queue: " + oldQueue.getQueuePath() - + " to parent queue since " - + "it is not yet in stopped state. Current State : " - + oldQueue.getState()); + "Can not convert the leaf queue: " + oldQueue.getQueuePath() + + " to parent queue since " + + "it is not yet in stopped state. Current State : " + + oldQueue.getState()); } } else if (oldQueue instanceof ParentQueue - && newQueue instanceof LeafQueue) { + && newQueue instanceof LeafQueue) { LOG.info("Converting the parent queue: " + oldQueue.getQueuePath() - + " to leaf queue."); + + " to leaf queue."); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index d57e3919cdd..c985ec9276f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -1049,7 +1050,10 @@ public void setOverrideWithQueueMappings(boolean overrideWithQueueMappings) { "Illegal queue mapping " + mappingValue); } - QueueMappingEntity m = new QueueMappingEntity(mapping[0], mapping[1]); + //Mappings should be consistent, and have the parent path parsed from the beginning + QueueMappingEntity m = new QueueMappingEntity( + mapping[0], + QueuePlacementRuleUtils.extractQueuePath(mapping[1])); mappings.add(m); } @@ -1120,10 +1124,12 @@ public boolean getOverrideWithWorkflowPriorityMappings() { throw new IllegalArgumentException( "unknown mapping prefix " + mapping[0]); } + //forcing the queue path to be split to parent and leafQueue, to make + //queue mapping parentPath and queueName consistent m = QueueMappingBuilder.create() .type(mappingType) .source(mapping[1]) - .queue(mapping[2]) + .queuePath(QueuePlacementRuleUtils.extractQueuePath(mapping[2])) .build(); } catch (Throwable t) { throw new IllegalArgumentException( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 1bbc7ca891e..74d3d804daf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,22 +64,6 @@ private static final Logger LOG = LoggerFactory.getLogger( CapacitySchedulerQueueManager.class); - static final Comparator NON_PARTITIONED_QUEUE_COMPARATOR = - new Comparator() { - @Override - public int compare(CSQueue q1, CSQueue q2) { - int result = Float.compare(q1.getUsedCapacity(), - q2.getUsedCapacity()); - if (result < 0) { - return -1; - } else if (result > 0) { - return 1; - } - - return q1.getQueuePath().compareTo(q2.getQueuePath()); - } - }; - static class QueueHook { public CSQueue hook(CSQueue queue) { return queue; @@ -90,7 +73,7 @@ public CSQueue hook(CSQueue queue) { private static final QueueHook NOOP = new QueueHook(); private CapacitySchedulerContext csContext; private final YarnAuthorizationProvider authorizer; - private final Map queues = new ConcurrentHashMap<>(); + private final CSQueueStore queues = new CSQueueStore(); private CSQueue root; private final RMNodeLabelsManager labelManager; private AppPriorityACLsManager appPriorityACLManager; @@ -120,7 +103,11 @@ public CSQueue getRootQueue() { @Override public Map getQueues() { - return queues; + return queues.getFullNameQueues(); + } + + public Map getShortNameQueues() { + return queues.getShortNameQueues(); } @Override @@ -130,7 +117,7 @@ public void removeQueue(String queueName) { @Override public void addQueue(String queueName, CSQueue queue) { - this.queues.put(queueName, queue); + this.queues.add(queue); } @Override @@ -138,6 +125,28 @@ public CSQueue getQueue(String queueName) { return queues.get(queueName); } + public CSQueue getQueueByFullName(String name) { + return queues.getByFullName(name); + } + + public CSQueue getQueueByShortName(String name) { + return queues.getByShortName(name); + } + + public String normalizeQueueName(String name) { + CSQueue queue = this.queues.get(name); + if (queue != null) { + return queue.getQueuePath(); + } + //We return the original name here instead of null, to make sure we don't introduce a NPE, and let the process + //fail where it would fail for unknown queues, resulting more informative error messages. + return name; + } + + public boolean isAmbiguous(String shortName) { + return queues.isAmbiguous(shortName); + } + /** * Set the CapacitySchedulerContext. * @param capacitySchedulerContext the CapacitySchedulerContext @@ -166,7 +175,7 @@ public void initializeQueues(CapacitySchedulerConfiguration conf) public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues - Map newQueues = new HashMap<>(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); @@ -212,8 +221,9 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) static CSQueue parseQueue( CapacitySchedulerContext csContext, CapacitySchedulerConfiguration conf, - CSQueue parent, String queueName, Map queues, - Map oldQueues, + CSQueue parent, String queueName, + CSQueueStore queues, + CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; String fullQueueName = (parent == null) ? @@ -232,7 +242,7 @@ static CSQueue parseQueue( // system if (isReservableQueue) { queue = new PlanQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); //initializing the "internal" default queue, for SLS compatibility String defReservationId = @@ -248,31 +258,31 @@ static CSQueue parseQueue( } childQueues.add(resQueue); ((PlanQueue) queue).setChildQueues(childQueues); - queues.put(defReservationId, resQueue); + queues.add(resQueue); } else if (isAutoCreateEnabled) { queue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ queue = new LeafQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); // Used only for unit tests queue = hook.hook(queue); } } else{ if (isReservableQueue) { throw new IllegalStateException( - "Only Leaf Queues can be reservable for " + queueName); + "Only Leaf Queues can be reservable for " + fullQueueName); } ParentQueue parentQueue; if (isAutoCreateEnabled) { parentQueue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ parentQueue = new ParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } // Used only for unit tests @@ -288,18 +298,17 @@ static CSQueue parseQueue( } - if (queue instanceof LeafQueue && queues.containsKey(queueName) && queues - .get(queueName) instanceof LeafQueue) { - throw new IOException("Two leaf queues were named " + queueName + if (queue instanceof LeafQueue && queues.get(fullQueueName) != null && queues + .get(fullQueueName) instanceof LeafQueue) { + throw new IOException("Two leaf queues were named " + fullQueueName + ". Leaf queue names must be distinct"); } - queues.put(queueName, queue); + queues.add(queue); - LOG.info("Initialized queue: " + queue); + LOG.info("Initialized queue: " + fullQueueName); return queue; } - /** * Updates to our list of queues: Adds the new queues and deletes the removed * ones... be careful, do not overwrite existing queues. @@ -307,29 +316,24 @@ static CSQueue parseQueue( * @param existingQueues, the existing queues * @param newQueues the new queues based on new XML */ - private void updateQueues(Map existingQueues, - Map newQueues) { + private void updateQueues(CSQueueStore existingQueues, + CSQueueStore newQueues) { CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - for (Map.Entry e : newQueues.entrySet()) { - String queueName = e.getKey(); - CSQueue queue = e.getValue(); - if (!existingQueues.containsKey(queueName)) { - existingQueues.put(queueName, queue); + for (CSQueue queue : newQueues.getQueues()) { + if (existingQueues.get(queue.getQueuePath()) == null) { + existingQueues.add(queue); } } - for (Iterator> itr = existingQueues.entrySet() - .iterator(); itr.hasNext();) { - Map.Entry e = itr.next(); - String queueName = e.getKey(); - CSQueue existingQueue = e.getValue(); - - if (!newQueues.containsKey(queueName) && !( - existingQueue instanceof AutoCreatedLeafQueue && conf + + for (CSQueue queue: existingQueues.getQueues()) { + if (newQueues.get(queue.getQueuePath()) == null && !( + queue instanceof AutoCreatedLeafQueue && conf .isAutoCreateChildQueueEnabled( - existingQueue.getParent().getQueuePath()))) { - itr.remove(); + queue.getParent().getQueuePath()))) { + existingQueues.remove(queue); } } + } @VisibleForTesting @@ -340,10 +344,10 @@ private void updateQueues(Map existingQueues, * @throws IOException if fails to set queue acls */ public static void setQueueAcls(YarnAuthorizationProvider authorizer, - AppPriorityACLsManager appPriorityACLManager, Map queues) + AppPriorityACLsManager appPriorityACLManager, CSQueueStore queues) throws IOException { List permissions = new ArrayList<>(); - for (CSQueue queue : queues.values()) { + for (CSQueue queue : queues.getQueues()) { AbstractCSQueue csQueue = (AbstractCSQueue) queue; permissions.add( new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs())); @@ -352,9 +356,9 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer, LeafQueue lQueue = (LeafQueue) queue; // Clear Priority ACLs first since reinitialize also call same. - appPriorityACLManager.clearPriorityACLs(lQueue.getQueueName()); + appPriorityACLManager.clearPriorityACLs(lQueue.getQueuePath()); appPriorityACLManager.addPrioirityACLs(lQueue.getPriorityACLs(), - lQueue.getQueueName()); + lQueue.getQueuePath()); } } authorizer.setPermission(permissions, @@ -406,7 +410,7 @@ public Priority getDefaultPriorityForQueue(String queueName) { private Map> getQueueToLabels() { Map> queueToLabels = new HashMap<>(); for (CSQueue queue : getQueues().values()) { - queueToLabels.put(queue.getQueueName(), queue.getAccessibleNodeLabels()); + queueToLabels.put(queue.getQueuePath(), queue.getAccessibleNodeLabels()); } return queueToLabels; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 37549280164..07d9e7c978a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -212,7 +212,7 @@ protected void setupQueueConfigs(Resource clusterResource, if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, this.defaultLabelExpression, null)) { throw new IOException( - "Invalid default label expression of " + " queue=" + getQueueName() + "Invalid default label expression of " + " queue=" + getQueuePath() + " doesn't have permission to access all labels " + "in default label expression. labelExpression of resource request=" + (this.defaultLabelExpression == null ? @@ -266,7 +266,7 @@ protected void setupQueueConfigs(Resource clusterResource, usersManager.updateUserWeights(); LOG.info( - "Initializing " + queueName + "\n" + "capacity = " + queueCapacities + "Initializing " + getQueuePath() + "\n" + "capacity = " + queueCapacities .getCapacity() + " [= (float) configuredCapacity / 100 ]" + "\n" + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() + " [= parentAbsoluteCapacity * capacity ]" + "\n" @@ -468,7 +468,7 @@ public QueueInfo getQueueInfo( } } - userAclInfo.setQueueName(getQueueName()); + userAclInfo.setQueueName(getQueuePath()); userAclInfo.setUserAcls(operations); return Collections.singletonList(userAclInfo); } finally { @@ -480,7 +480,7 @@ public QueueInfo getQueueInfo( public String toString() { readLock.lock(); try { - return queueName + ": " + "capacity=" + queueCapacities.getCapacity() + return getQueuePath() + ": " + "capacity=" + queueCapacities.getCapacity() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + "usedResources=" + queueUsage.getUsed() + ", " + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity=" @@ -763,7 +763,7 @@ public Resource calculateAndGetAMResourceLimitPerPartition( queueUsage.setAMLimit(nodePartition, amResouceLimit); LOG.debug("Queue: {}, node label : {}, queue partition resource : {}," + " queue current limit : {}, queue partition usable resource : {}," - + " amResourceLimit : {}", getQueueName(), nodePartition, + + " amResourceLimit : {}", getQueuePath(), nodePartition, queuePartitionResource, queueCurrentLimit, queuePartitionUsableResource, amResouceLimit); return amResouceLimit; @@ -880,7 +880,7 @@ protected void activateApplications() { application.getUser(), userAMLimit); fsApp.remove(); LOG.info("Application " + applicationId + " from user: " + application - .getUser() + " activated in queue: " + getQueueName()); + .getUser() + " activated in queue: " + getQueuePath()); } } finally { writeLock.unlock(); @@ -912,7 +912,7 @@ private void addApplicationAttempt(FiCaSchedulerApp application, LOG.info( "Application added -" + " appId: " + application.getApplicationId() + " user: " + application.getUser() + "," + " leaf-queue: " - + getQueueName() + " #user-pending-applications: " + user + + getQueuePath() + " #user-pending-applications: " + user .getPendingApplications() + " #user-active-applications: " + user .getActiveApplications() + " #queue-pending-applications: " + getNumPendingApplications() + " #queue-active-applications: " @@ -973,7 +973,7 @@ private void removeApplicationAttempt( LOG.info( "Application removed -" + " appId: " + application.getApplicationId() - + " user: " + application.getUser() + " queue: " + getQueueName() + + " user: " + application.getUser() + " queue: " + getQueuePath() + " #user-pending-applications: " + user.getPendingApplications() + " #user-active-applications: " + user.getActiveApplications() + " #queue-pending-applications: " + getNumPendingApplications() @@ -1056,7 +1056,7 @@ public CSAssignment assignContainers(Resource clusterResource, if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY && !accessibleToPartition(candidates.getPartition())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); return CSAssignment.NULL_ASSIGNMENT; } @@ -1072,7 +1072,7 @@ public CSAssignment assignContainers(Resource clusterResource, .getPartition()); } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); return CSAssignment.NULL_ASSIGNMENT; } @@ -1099,7 +1099,7 @@ public CSAssignment assignContainers(Resource clusterResource, activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); return CSAssignment.NULL_ASSIGNMENT; @@ -1162,7 +1162,7 @@ public CSAssignment assignContainers(Resource clusterResource, if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), + getParent().getQueuePath(), getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); return assignment; } else if (assignment.getSkippedType() @@ -1174,7 +1174,7 @@ public CSAssignment assignContainers(Resource clusterResource, } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + " from " + application.getApplicationId()); return assignment; @@ -1182,7 +1182,7 @@ public CSAssignment assignContainers(Resource clusterResource, // If we don't allocate anything, and it is not skipped by application, // we will return to respect FIFO of applications ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); ActivitiesLogger.APP.finishSkippedAppAllocationRecording( activitiesManager, application.getApplicationId(), @@ -1191,7 +1191,7 @@ public CSAssignment assignContainers(Resource clusterResource, } } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); return CSAssignment.NULL_ASSIGNMENT; @@ -1545,7 +1545,7 @@ protected boolean canAssignToUser(Resource clusterResource, application.getCurrentReservation()), limit)) { if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueueName() + LOG.debug("User " + userName + " in queue " + getQueuePath() + " will exceed limit based on reservations - " + " consumed: " + user.getUsed() + " reserved: " + application .getCurrentReservation() + " limit: " + limit); @@ -1560,7 +1560,7 @@ protected boolean canAssignToUser(Resource clusterResource, } } if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueueName() + LOG.debug("User " + userName + " in queue " + getQueuePath() + " will exceed limit - " + " consumed: " + user .getUsed(nodePartition) + " limit: " + limit); } @@ -1673,7 +1673,7 @@ public void completedContainer(Resource clusterResource, // Notify PreemptionManager csContext.getPreemptionManager().removeKillableContainer( - new KillableContainer(rmContainer, node.getPartition(), queueName)); + new KillableContainer(rmContainer, node.getPartition(), getQueuePath())); // Update preemption metrics if exit status is PREEMPTED if (containerStatus != null @@ -1717,7 +1717,7 @@ void allocateResource(Resource clusterResource, userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { - LOG.debug(getQueueName() + " user=" + userName + " used=" + LOG.debug(getQueuePath() + " user=" + userName + " used=" + queueUsage.getUsed(nodePartition) + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed()); @@ -1758,7 +1758,7 @@ void releaseResource(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug( - getQueueName() + " used=" + queueUsage.getUsed() + " numContainers=" + getQueuePath() + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " user=" + userName + " user-resources=" + user.getUsed()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 6c40a23f448..d7607ceb947 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -121,7 +121,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) queueName, super.getCapacity(), super.getMaximumCapacity()); } catch (YarnException ye) { LOG.error("Exception while computing policy changes for leaf queue : " - + getQueueName(), ye); + + getQueuePath(), ye); throw new IOException(ye); } finally { writeLock.unlock(); @@ -196,7 +196,7 @@ public void addChildQueue(CSQueue childQueue) ManagedParentQueue parentQueue = (ManagedParentQueue) childQueue.getParent(); - String leafQueueName = childQueue.getQueueName(); + String leafQueueName = childQueue.getQueuePath(); int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit( parentQueue.getQueuePath()); @@ -320,11 +320,11 @@ public void validateQueueManagementChanges( if (!(AbstractManagedParentQueue.class. isAssignableFrom(childQueue.getParent().getClass()))) { - LOG.error("Queue " + getQueueName() + LOG.error("Queue " + getQueuePath() + " is not an instance of PlanQueue or ManagedParentQueue." + " " + "Ignoring update " + queueManagementChanges); throw new SchedulerDynamicEditException( - "Queue " + getQueueName() + " is not a AutoEnabledParentQueue." + "Queue " + getQueuePath() + " is not a AutoEnabledParentQueue." + " Ignoring update " + queueManagementChanges); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 8d3244722cf..23250979933 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -182,8 +182,8 @@ void setChildQueues(Collection childQueues) { if (!queueName.equals("root") && getCapacity() != 0f && !queue.getQueueResourceQuotas().getConfiguredMinResource() .equals(Resources.none())) { - throw new IllegalArgumentException("Parent queue '" + getQueueName() - + "' and child queue '" + queue.getQueueName() + throw new IllegalArgumentException("Parent queue '" + getQueuePath() + + "' and child queue '" + queue.getQueuePath() + "' should use either percentage based capacity" + " configuration or absolute resource together."); } @@ -213,8 +213,8 @@ void setChildQueues(Collection childQueues) { // back an exception if (!queueName.equals("root") && !this.capacityConfigType .equals(queue.getCapacityConfigType())) { - throw new IllegalArgumentException("Parent queue '" + getQueueName() - + "' and child queue '" + queue.getQueueName() + throw new IllegalArgumentException("Parent queue '" + getQueuePath() + + "' and child queue '" + queue.getQueuePath() + "' should use either percentage based capacity" + "configuration or absolute resource together for label:" + nodeLabel); @@ -290,7 +290,7 @@ private QueueUserACLInfo getUserAclInfo( } } - userAclInfo.setQueueName(getQueueName()); + userAclInfo.setQueueName(getQueuePath()); userAclInfo.setUserAcls(operations); return userAclInfo; } finally { @@ -383,7 +383,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } // Re-init existing queues childQueue.reinitialize(newChildQueue, clusterResource); - LOG.info(getQueueName() + ": re-configured queue: " + childQueue); + LOG.info(getQueuePath() + ": re-configured queue: " + childQueue); } else{ // New child queue, do not re-init @@ -394,7 +394,7 @@ public void reinitialize(CSQueue newlyParsedQueue, currentChildQueues.put(newChildQueueName, newChildQueue); LOG.info( - getQueueName() + ": added new child queue: " + newChildQueue); + getQueuePath() + ": added new child queue: " + newChildQueue); } } @@ -422,7 +422,7 @@ public void reinitialize(CSQueue newlyParsedQueue, private Map getQueuesMap(List queues) { Map queuesMap = new HashMap(); for (CSQueue queue : queues) { - queuesMap.put(queue.getQueueName(), queue); + queuesMap.put(queue.getQueuePath(), queue); } return queuesMap; } @@ -499,7 +499,7 @@ private void addApplication(ApplicationId applicationId, LOG.info( "Application added -" + " appId: " + applicationId + " user: " + user - + " leaf-queue of parent: " + getQueueName() + " #applications: " + + " leaf-queue of parent: " + getQueuePath() + " #applications: " + getNumApplications()); } finally { writeLock.unlock(); @@ -526,7 +526,7 @@ private void removeApplication(ApplicationId applicationId, --numApplications; LOG.info("Application removed -" + " appId: " + applicationId + " user: " - + user + " leaf-queue of parent: " + getQueueName() + + user + " leaf-queue of parent: " + getQueuePath() + " #applications: " + getNumApplications()); } finally { writeLock.unlock(); @@ -534,7 +534,7 @@ private void removeApplication(ApplicationId applicationId, } private String getParentName() { - return getParent() != null ? getParent().getQueueName() : ""; + return getParent() != null ? getParent().getQueuePath() : ""; } @Override @@ -558,7 +558,7 @@ public CSAssignment assignContainers(Resource clusterResource, } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -585,7 +585,7 @@ public CSAssignment assignContainers(Resource clusterResource, } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.SKIPPED, + getParentName(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -600,7 +600,7 @@ public CSAssignment assignContainers(Resource clusterResource, while (canAssign(clusterResource, node)) { LOG.debug("Trying to assign containers to child-queue of {}", - getQueueName()); + getQueuePath()); // Are we over maximum-capacity for this queue? // This will also consider parent's limits and also continuous reservation @@ -612,7 +612,7 @@ public CSAssignment assignContainers(Resource clusterResource, getMetrics().getReservedVirtualCores()), schedulingMode)) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -640,7 +640,7 @@ public CSAssignment assignContainers(Resource clusterResource, assignedToChild.getResource(), Resources.none())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.ACCEPTED, + getParentName(), getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); boolean isReserved = @@ -678,12 +678,12 @@ public CSAssignment assignContainers(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("assignedContainer reserved=" + isReserved + " queue=" - + getQueueName() + " usedCapacity=" + getUsedCapacity() + + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); LOG.debug( - "ParentQ=" + getQueueName() + " assignedSoFarInThisIteration=" + "ParentQ=" + getQueuePath() + " assignedSoFarInThisIteration=" + assignment.getResource() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity()); @@ -692,7 +692,7 @@ public CSAssignment assignContainers(Resource clusterResource, assignment.setSkippedType(assignedToChild.getSkippedType()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.SKIPPED, + getParentName(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -731,7 +731,7 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) { node.getTotalKillableResources()), minimumAllocation); if (!accept) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, () -> node.getReservedContainer() != null ? ActivityDiagnosticConstant. QUEUE_SKIPPED_BECAUSE_SINGLE_NODE_RESERVED : @@ -832,9 +832,9 @@ private CSAssignment assignContainersToChildQueues(Resource cluster, limits.addBlockedHeadroom(resourceToSubtract); if(LOG.isDebugEnabled()) { LOG.debug("Decrease parentLimits " + limits.getLimit() + - " for " + this.getQueueName() + " by " + + " for " + this.getQueuePath() + " by " + resourceToSubtract + " as childQueue=" + - childQueue.getQueueName() + " is blocked"); + childQueue.getQueuePath() + " is blocked"); } } } @@ -938,7 +938,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, // cluster resource. Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); - if (getQueueName().equals("root")) { + if (getQueuePath().equals("root")) { queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel); queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel); queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); @@ -959,7 +959,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, // min_resources. Resource numeratorForMinRatio = null; ResourceCalculator rc = this.csContext.getResourceCalculator(); - if (getQueueName().equals("root")) { + if (getQueuePath().equals("root")) { if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc, clusterResource, resourceByLabel, configuredMinResources)) { numeratorForMinRatio = resourceByLabel; @@ -985,7 +985,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, if (childQueue.getCapacityConfigType() .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, - getMinResourceNormalized(childQueue.getQueueName(), effectiveMinRatioPerResource, + getMinResourceNormalized(childQueue.getQueuePath(), effectiveMinRatioPerResource, minResource)); // Max resource of a queue should be a minimum of {configuredMaxRes, @@ -1025,7 +1025,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, if (LOG.isDebugEnabled()) { LOG.debug("Updating effective min resource for queue:" - + childQueue.getQueueName() + " as effMinResource=" + + childQueue.getQueuePath() + " as effMinResource=" + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label) + "and Updating effective max resource as effMaxResource=" + childQueue.getQueueResourceQuotas() @@ -1129,7 +1129,7 @@ private void deriveCapacityFromAbsoluteConfigurations(String label, * (leafQueue.getUsersManager().getUserLimit() / 100.0f) * leafQueue.getUsersManager().getUserLimitFactor())); leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); - LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications=" + LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" + maxApplications + ", maxApplicationsPerUser=" + maxApplicationsPerUser + ", Abs Cap:" + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); @@ -1201,7 +1201,7 @@ public void attachContainer(Resource clusterResource, scheduler.getNode(rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, rmContainer.getContainer() .getResource(), node.getPartition()); - LOG.info("movedContainer" + " queueMoveIn=" + getQueueName() + LOG.info("movedContainer" + " queueMoveIn=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); @@ -1221,7 +1221,7 @@ public void detachContainer(Resource clusterResource, super.releaseResource(clusterResource, rmContainer.getContainer().getResource(), node.getPartition()); - LOG.info("movedContainer" + " queueMoveOut=" + getQueueName() + LOG.info("movedContainer" + " queueMoveOut=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); @@ -1316,8 +1316,8 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL, null, false); LOG.info("Killed container=" + toKillContainer.getContainerId() - + " from queue=" + lq.getQueueName() + " to make queue=" + this - .getQueueName() + "'s max-capacity enforced"); + + " from queue=" + lq.getQueuePath() + " to make queue=" + this + .getQueuePath() + "'s max-capacity enforced"); } if (!killableContainerIter.hasNext()) { @@ -1343,7 +1343,7 @@ public void apply(Resource cluster, allocateResource(cluster, allocation.getAllocatedOrReservedResource(), schedulerContainer.getNodePartition()); - LOG.info("assignedContainer" + " queue=" + getQueueName() + LOG.info("assignedContainer" + " queue=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + cluster); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java index 64ba578ff73..9e223e4153d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java @@ -124,7 +124,7 @@ public int hashCode() { @Override public String toString() { - return "QueueManagementChange{" + "queue=" + queue.getQueueName() + return "QueueManagementChange{" + "queue=" + queue.getQueuePath() + ", updatedEntitlementsByPartition=" + queueTemplateUpdate + ", queueAction=" + queueAction + ", transitionToQueueState=" + transitionToQueueState + '}'; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java index 96025588487..e6527ae4aa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java @@ -208,7 +208,7 @@ public void editSchedule() { policyClazz.getClass().getName(), clock.getTime() - startTime); if (queueManagementChanges.size() > 0) { LOG.debug(" Updated queue management changes for parent queue" + " " - + "{}: [{}]", parentQueue.getQueueName(), + + "{}: [{}]", parentQueue.getQueuePath(), queueManagementChanges.size() < 25 ? queueManagementChanges.toString() : queueManagementChanges.size()); @@ -218,7 +218,7 @@ public void editSchedule() { LOG.error( "Could not compute child queue management updates for parent " + "queue " - + parentQueue.getQueueName(), e); + + parentQueue.getQueuePath(), e); } } else{ LOG.debug("Skipping queue management updates for parent queue {} " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index b0700e8bf41..00b3a77d6fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -789,7 +789,7 @@ private Resource computeUserLimit(String userName, Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("User limit computation for " + userName - + ", in queue: " + lQueue.getQueueName() + + ", in queue: " + lQueue.getQueuePath() + ", userLimitPercent=" + lQueue.getUserLimit() + ", userLimitFactor=" + lQueue.getUserLimitFactor() + ", required=" + required diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java index 6b6d4faee16..70387477532 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/WorkflowPriorityMappingsManager.java @@ -196,7 +196,7 @@ public Priority mapWorkflowPriorityForApp(ApplicationId applicationId, Priority mappedPriority = getMappedPriority(workflowID, queue); if (mappedPriority != null) { LOG.info("Application " + applicationId + " user " + user - + " workflow " + workflowID + " queue " + queue.getQueueName() + + " workflow " + workflowID + " queue " + queue.getQueuePath() + " mapping [" + priority + "] to [" + mappedPriority + "] override " + overrideWithPriorityMappings); @@ -206,7 +206,7 @@ public Priority mapWorkflowPriorityForApp(ApplicationId applicationId, priority = mappedPriority; priority = scheduler.checkAndGetApplicationPriority( priority, UserGroupInformation.createRemoteUser(user), - queue.getQueueName(), applicationId); + queue.getQueuePath(), applicationId); rmApp.getApplicationSubmissionContext().setPriority(priority); ((RMAppImpl)rmApp).setApplicationPriority(priority); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java index ae070872074..58110c9b691 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java @@ -47,11 +47,11 @@ public void refreshQueues(CSQueue parent, CSQueue current) { try { PreemptableQueue parentEntity = null; if (parent != null) { - parentEntity = entities.get(parent.getQueueName()); + parentEntity = entities.get(parent.getQueuePath()); } - if (!entities.containsKey(current.getQueueName())) { - entities.put(current.getQueueName(), + if (!entities.containsKey(current.getQueuePath())) { + entities.put(current.getQueuePath(), new PreemptableQueue(parentEntity)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index d91f4887db4..614b8242aba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -136,7 +136,7 @@ private boolean addLeafQueueStateIfNotExists(String leafQueueName, public boolean createLeafQueueStateIfNotExists(LeafQueue leafQueue, String partition) { - return addLeafQueueStateIfNotExists(leafQueue.getQueueName(), partition, + return addLeafQueueStateIfNotExists(leafQueue.getQueuePath(), partition, new LeafQueueStatePerPartition()); } @@ -311,7 +311,7 @@ public void init(final CapacitySchedulerContext schedulerContext, LOG.info( "Initialized queue management policy for parent queue " + parentQueue - .getQueueName() + " with leaf queue template capacities : [" + .getQueuePath() + " with leaf queue template capacities : [" + leafQueueTemplate.getQueueCapacities() + "]"); } @@ -328,10 +328,10 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) if (!parentQueueLabels.contains(nodeLabel)) { LOG.error("Invalid node label " + nodeLabel + " on configured leaf template on parent" + " queue " + parentQueue - .getQueueName()); + .getQueuePath()); throw new IOException("Invalid node label " + nodeLabel + " on configured leaf template on parent" + " queue " + parentQueue - .getQueueName()); + .getQueuePath()); } } @@ -386,7 +386,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) if ( deactivatedLeafQueues.size() > 0) { LOG.debug("Parent queue = {}, " + ", nodeLabel = {}, deactivated leaf queues = [{}] ", - managedParentQueue.getQueueName(), nodeLabel, + managedParentQueue.getQueuePath(), nodeLabel, deactivatedLeafQueues.size() > 25 ? deactivatedLeafQueues .size() : deactivatedLeafQueues); @@ -405,7 +405,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) + deactivatedCapacity + EPSILON; if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueueName() + LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() + ", nodeLabel = " + nodeLabel + ", absCapacity = " + parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = " + leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = " @@ -422,7 +422,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) pendingApps.size()); if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueueName() + LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() + " : Found " + maxLeafQueuesTobeActivated + " leaf queues" + " to be activated with " + pendingApps.size() + " apps "); } @@ -495,7 +495,7 @@ void updateLeafQueueState() { nodeLabel); newPartitions.add(nodeLabel); } - newQueues.add(newQueue.getQueueName()); + newQueues.add(newQueue.getQueuePath()); } } @@ -507,7 +507,7 @@ void updateLeafQueueState() { String partition = e.getKey(); if (!newPartitions.contains(partition)) { itr.remove(); - LOG.info(managedParentQueue.getQueueName() + + LOG.info(managedParentQueue.getQueuePath() + " : Removed partition " + partition + " from leaf queue " + "state"); } else{ @@ -518,7 +518,7 @@ void updateLeafQueueState() { String queue = queueItr.next().getKey(); if (!newQueues.contains(queue)) { queueItr.remove(); - LOG.info(managedParentQueue.getQueueName() + " : Removed queue" + LOG.info(managedParentQueue.getQueuePath() + " : Removed queue" + queue + " from " + "leaf queue " + "state from partition " + partition); @@ -540,7 +540,7 @@ void updateLeafQueueState() { for (FiCaSchedulerApp app : pendingApps) { AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) app.getCSLeafQueue(); - String leafQueueName = leafQueue.getQueueName(); + String leafQueueName = leafQueue.getQueuePath(); //Check if leafQueue is not active already and has any pending apps if (ctr < leafQueuesNeeded) { @@ -590,15 +590,15 @@ public boolean isActive(final AutoCreatedLeafQueue leafQueue, AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue; if (leafQueue != null) { if (isActive(leafQueue, nodeLabel) && !hasPendingApps(leafQueue)) { - if (!leafQueueEntitlements.containsKey(leafQueue.getQueueName())) { - leafQueueEntitlements.put(leafQueue.getQueueName(), + if (!leafQueueEntitlements.containsKey(leafQueue.getQueuePath())) { + leafQueueEntitlements.put(leafQueue.getQueuePath(), new QueueCapacities(false)); } QueueCapacities capacities = leafQueueEntitlements.get( - leafQueue.getQueueName()); + leafQueue.getQueuePath()); updateToZeroCapacity(capacities, nodeLabel); - deactivatedQueues.put(leafQueue.getQueueName(), + deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities); } } else{ @@ -670,14 +670,14 @@ public void commitQueueManagementChanges( getCapacity(nodeLabel) > 0) { if (isActive(leafQueue, nodeLabel)) { LOG.debug("Queue is already active. Skipping activation : {}", - leafQueue.getQueueName()); + leafQueue.getQueuePath()); } else{ activate(leafQueue, nodeLabel); } } else{ if (!isActive(leafQueue, nodeLabel)) { LOG.debug("Queue is already de-activated. Skipping " - + "de-activation : {}", leafQueue.getQueueName()); + + "de-activation : {}", leafQueue.getQueuePath()); } else{ deactivate(leafQueue, nodeLabel); } @@ -744,7 +744,7 @@ public void reinitialize(CapacitySchedulerContext schedulerContext, LOG.info( "Reinitialized queue management policy for parent queue " + parentQueue - .getQueueName() + " with leaf queue template " + "capacities : [" + .getQueuePath() + " with leaf queue template " + "capacities : [" + leafQueueTemplate.getQueueCapacities() + "]"); } @@ -813,7 +813,7 @@ LeafQueueStatePerPartition getLeafQueueState(LeafQueue queue, String partition) throws SchedulerDynamicEditException { readLock.lock(); try { - String queueName = queue.getQueueName(); + String queueName = queue.getQueuePath(); if (!leafQueueState.containsLeafQueue(queueName, partition)) { throw new SchedulerDynamicEditException( "Could not find leaf queue in " + "state " + queueName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 0789bfd2422..f42351c85ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -273,7 +273,12 @@ public RMContainer allocate(FiCaSchedulerNode node, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, ps.getPrimaryRequestedNodePartition()); - ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName()); + + String qn = this.getQueueName(); + if (this.scheduler instanceof CapacityScheduler) { + qn = ((CapacityScheduler)this.scheduler).normalizeQueueName(qn); + } + ((RMContainerImpl) rmContainer).setQueueName(qn); // FIXME, should set when confirmed updateAMContainerDiagnostics(AMState.ASSIGNED, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java index 1caa181176e..065b92ff87f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -67,6 +67,13 @@ public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, if (scheduler instanceof CapacityScheduler) { CSQueue queue = ((CapacityScheduler) scheduler).getQueue(app.getQueue()); if (queue == null) { + if (((CapacityScheduler) scheduler).isAmbiguous(app.getQueue())) { + LOG.error("Queue " + app.getQueue() + " is ambiguous for " + app + .getApplicationId()); + //if we cannot decide which queue to submit we should deny access + return false; + } + // The application exists but the associated queue does not exist. // This may be due to a queue that is not defined when the RM restarts. // At this point we choose to log the fact and allow users to access @@ -116,11 +123,14 @@ public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, // extra logging to distinguish between the queue not existing in the // application move request case and the real access denied case. if (scheduler instanceof CapacityScheduler) { - CSQueue queue = ((CapacityScheduler) scheduler).getQueue(targetQueue); + CapacityScheduler cs = ((CapacityScheduler) scheduler); + CSQueue queue = cs.getQueue(targetQueue); if (queue == null) { LOG.warn("Target queue " + targetQueue - + " does not exist while trying to move " - + app.getApplicationId()); + + (cs.isAmbiguous(targetQueue) ? + " is ambiguous while trying to move " : + " does not exist while trying to move ") + + app.getApplicationId()); return false; } return authorizer.checkPermission( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java index f746dc2f188..239fbf3832f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java @@ -194,7 +194,7 @@ private void addNodeCapacityToPlan(MockRM rm, int memory, int vCores) { break; } LOG.info("Waiting for node capacity to be added to plan"); - Thread.sleep(100); + Thread.sleep(1000); } while (attempts-- > 0); if (attempts <= 0) { Assert.fail("Exhausted attempts in checking if node capacity was " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java index 4a762fa9b0f..a8b9e92bc93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java @@ -449,6 +449,10 @@ private void mockApplications(String appsConfig) { Resource capacity = Resources.multiply(totResoucePerPartition, queue.getQueueCapacities().getAbsoluteCapacity()); HashSet users = userMap.get(queue.getQueueName()); + //TODO: Refactor this test class to use queue path internally like CS does from now on + if (users == null) { + users = userMap.get(queue.getQueuePath()); + } when(queue.getAllUsers()).thenReturn(users); Resource userLimit; if (mulp > 0) { @@ -669,7 +673,7 @@ public Object answer(InvocationOnMock invocation) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); when(queue.getReadLock()).thenReturn(lock.readLock()); setupQueue(queue, q, queueExprArray, idx); - if (queue.getQueueName().equals(ROOT)) { + if (queue.getQueuePath().equals(ROOT)) { rootQueue = (ParentQueue) queue; } } @@ -684,7 +688,7 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, int myLevel = getLevel(q); if (0 == myLevel) { // It's root - when(queue.getQueueName()).thenReturn(ROOT); + when(queue.getQueuePath()).thenReturn(ROOT); queuePath = ROOT; } @@ -710,10 +714,10 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, when(queue.getQueueResourceUsage()).thenReturn(ru); when(queue.getQueueResourceQuotas()).thenReturn(qr); - LOG.debug("Setup queue, name=" + queue.getQueueName() + " path=" + LOG.debug("Setup queue, short name=" + queue.getQueueName() + " path=" + queue.getQueuePath()); LOG.debug("Parent=" + (parentQueue == null ? "null" : parentQueue - .getQueueName())); + .getQueuePath())); // Setup other fields like used resource, guaranteed resource, etc. String capacitySettingStr = q.substring(q.indexOf("(") + 1, q.indexOf(")")); @@ -796,8 +800,13 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, Boolean.valueOf(otherConfigs.get("disable_preemption"))); } + //TODO: Refactor this test class to use queue path internally like CS does from now on + nameToCSQueues.put(queuePath, queue); nameToCSQueues.put(queueName, queue); + when(cs.getQueue(eq(queuePath))).thenReturn(queue); when(cs.getQueue(eq(queueName))).thenReturn(queue); + when(cs.normalizeQueueName(eq(queuePath))).thenReturn(queuePath); + when(cs.normalizeQueueName(eq(queueName))).thenReturn(queuePath); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index c095573680a..da77f016137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -966,11 +966,11 @@ public void testHierarchicalLarge3Levels() { // which is likely triggered since we use small numbers for readability //run with Logger.getRootLogger().setLevel(Level.DEBUG); verify(mDisp, times(9)).handle(argThat(new IsPreemptionRequestFor(appC))); - assertEquals(10, policy.getQueuePartitions().get("queueE").get("").preemptableExtra.getMemorySize()); + assertEquals(10, policy.getQueuePartitions().get("root.queueA.queueC.queueE").get("").preemptableExtra.getMemorySize()); //2nd level child(E) preempts 10, but parent A has only 9 extra //check the parent can prempt only the extra from > 2 level child TempQueuePerPartition tempQueueAPartition = policy.getQueuePartitions().get( - "queueA").get(""); + "root.queueA").get(""); assertEquals(0, tempQueueAPartition.untouchableExtra.getMemorySize()); long extraForQueueA = tempQueueAPartition.getUsed().getMemorySize() - tempQueueAPartition @@ -999,12 +999,12 @@ public void testHierarchicalLarge3LevelsWithReserved() { policy.editSchedule(); verify(mDisp, times(9)).handle(argThat(new IsPreemptionRequestFor(appC))); - assertEquals(10, policy.getQueuePartitions().get("queueE") + assertEquals(10, policy.getQueuePartitions().get("root.queueA.queueC.queueE") .get("").preemptableExtra.getMemorySize()); //2nd level child(E) preempts 10, but parent A has only 9 extra //check the parent can prempt only the extra from > 2 level child TempQueuePerPartition tempQueueAPartition = policy.getQueuePartitions().get( - "queueA").get(""); + "root.queueA").get(""); assertEquals(0, tempQueueAPartition.untouchableExtra.getMemorySize()); long extraForQueueA = tempQueueAPartition.getUsed().getMemorySize() - tempQueueAPartition @@ -1189,7 +1189,7 @@ ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, ResourceUsage resUsage = new ResourceUsage(); resUsage.setUsed(used[0]); resUsage.setReserved(reserved[0]); - when(root.getQueueName()).thenReturn(CapacitySchedulerConfiguration.ROOT); + when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT); when(root.getAbsoluteUsedCapacity()).thenReturn( Resources.divide(rc, tot, used[0], tot)); when(root.getAbsoluteCapacity()).thenReturn( @@ -1231,7 +1231,7 @@ ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran); } when(q.getParent()).thenReturn(p); - when(q.getQueueName()).thenReturn(queueName); + when(q.getQueuePath()).thenReturn(queueName); when(q.getAbsoluteUsedCapacity()).thenReturn( Resources.divide(rc, tot, used[i], tot)); when(q.getAbsoluteCapacity()).thenReturn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java index 6b78c3944eb..582bf433cd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForReservedContainers.java @@ -70,10 +70,10 @@ public void testPreemptionForSimpleReservedContainer() throws IOException { "-b(=[50 100 10 9 9])"; // b String appsConfig= //queueName\t(priority,resource,host,expression,#repeat,reserved) - "a\t" // app1 in a + "root.a\t" // app1 in a + "(1,1,n1,,45,false)" // 45 in n1 + "(1,1,n2,,45,false);" + // 45 in n2 - "b\t" // app2 in b + "root.b\t" // app2 in b + "(1,1,n1,,1,false)" // AM container in n1 + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1 @@ -86,7 +86,7 @@ public void testPreemptionForSimpleReservedContainer() throws IOException { new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(1)))); verify(mDisp, times(5)).handle( - argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a", + argThat(new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "root.a", NodeId.newInstance("n1", 1)))); verify(mDisp, times(0)).handle(argThat( new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( @@ -296,10 +296,10 @@ public void testPreemptionForReservedContainerWhichHasAvailableResource() "-b(=[50 100 9 9 9])"; // b String appsConfig= //queueName\t(priority,resource,host,expression,#repeat,reserved) - "a\t" // app1 in a + "root.a\t" // app1 in a + "(1,1,n1,,45,false)" // 45 in n1 + "(1,1,n2,,45,false);" + // 45 in n2 - "b\t" // app2 in b + "root.b\t" // app2 in b + "(1,9,n1,,1,true)"; // 1 container with size=9 reserved at n1 buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); @@ -308,10 +308,10 @@ public void testPreemptionForReservedContainerWhichHasAvailableResource() // Total 4 preempted from app1 at n1, don't preempt container from other // app/node verify(mDisp, times(4)).handle(argThat( - new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a", + new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "root.a", NodeId.newInstance("n1", 1)))); verify(mDisp, times(0)).handle(argThat( - new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a", + new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "root.a", NodeId.newInstance("n2", 1)))); } @@ -349,7 +349,7 @@ public void testPreemptionForReservedContainerWhichHasNondivisibleAvailableResou "-b(=[50 100 9 9 9])"; // b String appsConfig= //queueName\t(priority,resource,host,expression,#repeat,reserved) - "a\t" // app1 in a + "root.a\t" // app1 in a + "(1,2,n1,,24,false)" // 48 in n1 + "(1,2,n2,,23,false);" + // 46 in n2 "b\t" // app2 in b @@ -361,10 +361,10 @@ public void testPreemptionForReservedContainerWhichHasNondivisibleAvailableResou // Total 4 preempted from app1 at n1, don't preempt container from other // app/node verify(mDisp, times(4)).handle(argThat( - new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a", + new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "root.a", NodeId.newInstance("n1", 1)))); verify(mDisp, times(0)).handle(argThat( - new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "a", + new IsPreemptionRequestForQueueAndNode(getAppAttemptId(1), "root.a", NodeId.newInstance("n2", 1)))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java index 584cb1fcca2..e2336fb1e44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java @@ -433,13 +433,13 @@ public void testAlreadySelectedContainerFromInterQueuePreemption() String appsConfig = // queueName\t(priority,resource,host,expression,#repeat,reserved,pending) - "a\t" // app1 in a + "root.a\t" // app1 in a + "(1,1,n1,,50,false,15);" + // app1 a - "a\t" // app2 in a + "root.a\t" // app2 in a + "(2,1,n1,,20,false,20);" + // app2 a - "b\t" // app3 in b + "root.b\t" // app3 in b + "(4,1,n1,,20,false,20);" + // app3 b - "b\t" // app1 in a + "root.b\t" // app1 in a + "(4,1,n1,,5,false,100)"; buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java index 8c0eee6865e..ec71e67f591 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java @@ -64,11 +64,11 @@ public void testPreemptionToBalanceDisabled() throws IOException { new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(3)))); - assertEquals(30, policy.getQueuePartitions().get("a") + assertEquals(30, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getMemorySize()); - assertEquals(35, policy.getQueuePartitions().get("b") + assertEquals(35, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getMemorySize()); - assertEquals(35, policy.getQueuePartitions().get("c") + assertEquals(35, policy.getQueuePartitions().get("root.c") .get("").getIdealAssigned().getMemorySize()); } @@ -106,11 +106,11 @@ public void testPreemptionToBalanceEnabled() throws IOException { new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(3)))); - assertEquals(33, policy.getQueuePartitions().get("a") + assertEquals(33, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getMemorySize()); - assertEquals(33, policy.getQueuePartitions().get("b") + assertEquals(33, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getMemorySize()); - assertEquals(33, policy.getQueuePartitions().get("c") + assertEquals(33, policy.getQueuePartitions().get("root.c") .get("").getIdealAssigned().getMemorySize()); } @@ -146,11 +146,11 @@ public void testPreemptionToBalanceUsedPlusPendingLessThanGuaranteed() new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(3)))); - assertEquals(16, policy.getQueuePartitions().get("a") + assertEquals(16, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getMemorySize()); - assertEquals(42, policy.getQueuePartitions().get("b") + assertEquals(42, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getMemorySize()); - assertEquals(42, policy.getQueuePartitions().get("c") + assertEquals(42, policy.getQueuePartitions().get("root.c") .get("").getIdealAssigned().getMemorySize()); } @@ -182,13 +182,13 @@ public void testPreemptionToBalanceWithVcoreResource() throws IOException { new TestProportionalCapacityPreemptionPolicy. IsPreemptionRequestFor(getAppAttemptId(2)))); - assertEquals(60, policy.getQueuePartitions().get("a") + assertEquals(60, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getMemorySize()); - assertEquals(60, policy.getQueuePartitions().get("a") + assertEquals(60, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getVirtualCores()); - assertEquals(40, policy.getQueuePartitions().get("b") + assertEquals(40, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getMemorySize()); - assertEquals(40, policy.getQueuePartitions().get("b") + assertEquals(40, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getVirtualCores()); } @@ -244,13 +244,13 @@ public void testPreemptionToBalanceWithConfiguredTimeout() throws IOException { new TestProportionalCapacityPreemptionPolicy. IsPreemptionRequestFor(getAppAttemptId(2)))); - assertEquals(60, policy.getQueuePartitions().get("a") + assertEquals(60, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getMemorySize()); - assertEquals(60, policy.getQueuePartitions().get("a") + assertEquals(60, policy.getQueuePartitions().get("root.a") .get("").getIdealAssigned().getVirtualCores()); - assertEquals(40, policy.getQueuePartitions().get("b") + assertEquals(40, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getMemorySize()); - assertEquals(40, policy.getQueuePartitions().get("b") + assertEquals(40, policy.getQueuePartitions().get("root.b") .get("").getIdealAssigned().getVirtualCores()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java index 88b7e68f83c..3ca7eb130c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java @@ -25,14 +25,20 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import java.util.Arrays; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestAppNameMappingPlacementRule { private static final String APP_NAME = "DistributedShell"; @@ -55,6 +61,11 @@ private void verifyQueueMapping(QueueMappingEntity queueMapping, boolean overwrite) throws YarnException { AppNameMappingPlacementRule rule = new AppNameMappingPlacementRule( overwrite, Arrays.asList(queueMapping)); + + CapacitySchedulerQueueManager qm = mock(CapacitySchedulerQueueManager.class); + when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false); + rule.queueManager = qm; + ApplicationSubmissionContext asc = Records.newRecord( ApplicationSubmissionContext.class); if (inputQueue.equals("%application")) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index e663a14c462..e5b4696a8b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -70,18 +70,18 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData) mock(CapacitySchedulerQueueManager.class); ParentQueue agroup = mock(ParentQueue.class); - when(agroup.getQueueName()).thenReturn("agroup"); + when(agroup.getQueuePath()).thenReturn("agroup"); ParentQueue bsubgroup2 = mock(ParentQueue.class); - when(bsubgroup2.getQueueName()).thenReturn("bsubgroup2"); + when(bsubgroup2.getQueuePath()).thenReturn("bsubgroup2"); LeafQueue a = mock(LeafQueue.class); - when(a.getQueueName()).thenReturn("a"); + when(a.getQueuePath()).thenReturn("a"); when(a.getParent()).thenReturn(agroup); LeafQueue b = mock(LeafQueue.class); - when(b.getQueueName()).thenReturn("b"); + when(b.getQueuePath()).thenReturn("b"); when(b.getParent()).thenReturn(bsubgroup2); LeafQueue asubgroup2 = mock(LeafQueue.class); - when(asubgroup2.getQueueName()).thenReturn("asubgroup2"); + when(asubgroup2.getQueuePath()).thenReturn("asubgroup2"); when(queueManager.getQueue("a")).thenReturn(a); when(queueManager.getQueue("b")).thenReturn(b); @@ -89,6 +89,17 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData) when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2); when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2); + when(queueManager.getQueue("root.agroup")).thenReturn(agroup); + when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2); + when(queueManager.getQueue("root.asubgroup2")).thenReturn(asubgroup2); + when(queueManager.getQueue("root.agroup.a")).thenReturn(a); + when(queueManager.getQueue("root.bsubgroup2.b")).thenReturn(b); + when(queueManager.getQueueByFullName("root.agroup")).thenReturn(agroup); + when(queueManager.getQueueByFullName("root.bsubgroup2")).thenReturn(bsubgroup2); + when(queueManager.getQueueByFullName("root.asubgroup2")).thenReturn(asubgroup2); + when(queueManager.getQueueByFullName("root.agroup.a")).thenReturn(a); + when(queueManager.getQueueByFullName("root.bsubgroup2.b")).thenReturn(b); + rule.setQueueManager(queueManager); ApplicationSubmissionContext asc = Records.newRecord( ApplicationSubmissionContext.class); @@ -200,7 +211,7 @@ public void testMapping() throws YarnException { .queue("%primary_group") .build()) .inputUser("a") - .expectedQueue("agroup") + .expectedQueue("root.agroup") .build()); verifyQueueMapping( QueueMappingTestDataBuilder.create() @@ -212,7 +223,7 @@ public void testMapping() throws YarnException { .build()) .inputUser("a") .expectedQueue("a") - .expectedParentQueue("agroup") + .expectedParentQueue("root.agroup") .build()); verifyQueueMapping( QueueMappingTestDataBuilder.create() @@ -224,7 +235,7 @@ public void testMapping() throws YarnException { .build()) .inputUser("b") .expectedQueue("b") - .expectedParentQueue("bsubgroup2") + .expectedParentQueue("root.bsubgroup2") .build()); verifyQueueMapping( QueueMappingTestDataBuilder.create() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 5dbb3deed48..1ee30119cc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -73,7 +73,8 @@ private static Random rand = new Random(); - public final static String reservationQ = "dedicated"; + public final static String reservationQShort = "dedicated"; + public final static String reservationQ = "root." + reservationQShort; public static ReservationId getNewReservationId() { return ReservationId.newInstance(rand.nextLong(), rand.nextLong()); @@ -296,13 +297,13 @@ public static void setupQueueConfiguration( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] { "default", "a", reservationQ }); + new String[] { "default", "a", reservationQShort }); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); final String dedicated = CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT + reservationQ; + + CapacitySchedulerConfiguration.DOT + reservationQShort; conf.setCapacity(dedicated, 80); // Set as reservation queue conf.setReservable(dedicated, true); @@ -319,9 +320,9 @@ public static void setupDynamicQueueConfiguration( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] { reservationQ }); + new String[] { reservationQShort }); final String dedicated = CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT + reservationQ; + + CapacitySchedulerConfiguration.DOT + reservationQShort; conf.setCapacity(dedicated, 100); // Set as reservation queue conf.setReservable(dedicated, true); @@ -329,7 +330,7 @@ public static void setupDynamicQueueConfiguration( public static String getFullReservationQueueName() { return CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT + reservationQ; + + CapacitySchedulerConfiguration.DOT + reservationQShort; } public static String getReservationQueueName() { @@ -346,12 +347,12 @@ public static void updateQueueConfiguration( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] { "default", "a", reservationQ, newQ }); + new String[] { "default", "a", reservationQShort, newQ }); final String A = prefix + "a"; conf.setCapacity(A, 5); - final String dedicated = prefix + reservationQ; + final String dedicated = prefix + reservationQShort; conf.setCapacity(dedicated, 10); // Set as reservation queue conf.setReservable(dedicated, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java index 6c4fac8e6cf..2622041afff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java @@ -123,13 +123,7 @@ public void testReinitialize() throws IOException { } catch (YarnException e) { Assert.fail(e.getMessage()); } - if (getSchedulerType().equals(SchedulerType.CAPACITY)) { - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, - newQ); - } else { - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, - "root." + newQ); - } + ReservationSystemTestUtil.validateReservationQueue(reservationSystem,"root." + newQ); } @SuppressWarnings("rawtypes") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index 48f6654c553..29a7939eb9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -72,6 +72,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -1072,7 +1074,11 @@ public void testContainerRecoveredByNode() throws Exception { RMContainer rmContainer = scheduler.getRMContainer(containerId); //verify queue name when rmContainer is recovered - Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName()); + if (scheduler instanceof CapacityScheduler) { + Assert.assertEquals(CapacitySchedulerConfiguration.ROOT + "." + app1.getQueue(), rmContainer.getQueueName()); + } else { + Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName()); + } } finally { rm1.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index fe4c880a56a..dc4d4d6a7b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -337,7 +337,7 @@ public void testSimpleValidateAbsoluteResourceConfig() throws Exception { Assert.assertEquals( "Failed to re-init queues : Min resource configuration " + " is greater than its " - + "max value: in queue:queueB1", + + "max value: in queue:root.queueB.queueB1", e.getMessage()); } @@ -358,7 +358,7 @@ public void testSimpleValidateAbsoluteResourceConfig() throws Exception { .assertEquals( "Failed to re-init queues : Max resource configuration " + " is greater than parents max value:" - + " in queue:queueB1", + + " in queue:root.queueB.queueB1", e.getMessage()); } rm.stop(); @@ -408,8 +408,8 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { } catch (IOException e) { Assert.assertTrue(e instanceof IOException); Assert.assertEquals( - "Failed to re-init queues : Parent queue 'queueA' " - + "and child queue 'queueA1' should use either percentage based" + "Failed to re-init queues : Parent queue 'root.queueA' " + + "and child queue 'root.queueA.queueA1' should use either percentage based" + " capacity configuration or absolute resource together.", e.getMessage()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 4ee73f2190e..4f35e0ac89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -128,7 +128,7 @@ public void setUp() throws IOException { when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); root = CapacitySchedulerQueueManager .parseQueue(csContext, csConf, null, "root", queues, queues, @@ -295,8 +295,8 @@ public void testLimitsComputation() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -368,7 +368,7 @@ public void testLimitsComputation() throws Exception { csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -392,7 +392,7 @@ public void testLimitsComputation() throws Exception { csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -603,8 +603,8 @@ public void testHeadroom() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index a5b9c38c6ff..9ff066d9ef5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -789,7 +789,7 @@ public void testHeadroom() throws Exception { Resource clusterResource = Resources.createResource(160 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java new file mode 100644 index 00000000000..04c03ff8b63 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java @@ -0,0 +1,13 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.junit.Test; + +import java.util.HashMap; + +import static org.junit.Assert.*; + +public class TestCSQueueStore { + @Test + public void testMapping() { + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index de772f11ec4..188f753f6f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2527,7 +2527,7 @@ public void testMoveAppQueueMetricsCheck() throws Exception { private int getNumAppsInQueue(String name, List queues) { for (CSQueue queue : queues) { - if (queue.getQueueName().equals(name)) { + if (queue.getQueueShortName().equals(name)) { return queue.getNumApplications(); } } @@ -5040,6 +5040,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { * Test for queue deletion. * @throws Exception */ + //TODO: Rethink this test to @Test public void testRefreshQueuesWithQueueDelete() throws Exception { CapacityScheduler cs = new CapacityScheduler(); @@ -5058,12 +5059,12 @@ public void testRefreshQueuesWithQueueDelete() throws Exception { // test delete leaf queue when there is application running. Map queues = - cs.getCapacitySchedulerQueueManager().getQueues(); + cs.getCapacitySchedulerQueueManager().getShortNameQueues(); String b1QTobeDeleted = "b1"; LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted)); when(csB1Queue.getState()).thenReturn(QueueState.DRAINING) .thenReturn(QueueState.STOPPED); - queues.put(b1QTobeDeleted, csB1Queue); + cs.getCapacitySchedulerQueueManager().addQueue(b1QTobeDeleted, csB1Queue); conf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithOutB1(conf); try { @@ -5100,23 +5101,23 @@ public void testRefreshQueuesWithQueueDelete() throws Exception { // set the configurations such that it fails once but should be successfull // next time - queues = cs.getCapacitySchedulerQueueManager().getQueues(); + queues = cs.getCapacitySchedulerQueueManager().getShortNameQueues(); CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b")); when(bQueue.getState()).thenReturn(QueueState.DRAINING) .thenReturn(QueueState.STOPPED); - queues.put("b", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b", bQueue); bQueue = Mockito.spy((LeafQueue) queues.get("b1")); when(bQueue.getState()).thenReturn(QueueState.STOPPED); - queues.put("b1", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); bQueue = Mockito.spy((LeafQueue) queues.get("b2")); when(bQueue.getState()).thenReturn(QueueState.STOPPED); - queues.put("b2", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); bQueue = Mockito.spy((LeafQueue) queues.get("b3")); when(bQueue.getState()).thenReturn(QueueState.STOPPED); - queues.put("b3", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); // test delete Parent queue when there is application running. conf = new CapacitySchedulerConfiguration(); @@ -5175,20 +5176,20 @@ public void testRefreshQueuesWithAllChildQueuesDeleted() throws Exception { // test delete all leaf queues when there is no application running. Map queues = - cs.getCapacitySchedulerQueueManager().getQueues(); + cs.getCapacitySchedulerQueueManager().getShortNameQueues(); CSQueue bQueue = Mockito.spy((LeafQueue) queues.get("b1")); when(bQueue.getState()).thenReturn(QueueState.RUNNING) .thenReturn(QueueState.STOPPED); - queues.put("b1", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); bQueue = Mockito.spy((LeafQueue) queues.get("b2")); when(bQueue.getState()).thenReturn(QueueState.STOPPED); - queues.put("b2", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); bQueue = Mockito.spy((LeafQueue) queues.get("b3")); when(bQueue.getState()).thenReturn(QueueState.STOPPED); - queues.put("b3", bQueue); + cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); conf = new CapacitySchedulerConfiguration(); setupQueueConfWithOutChildrenOfB(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index ae30e4a723b..c5eaa4c042b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -480,8 +480,8 @@ protected void cleanupQueue(String queueName) throws YarnException { if (queue != null) { setEntitlement(queue, new QueueEntitlement(0.0f, 0.0f)); ((ManagedParentQueue) queue.getParent()).removeChildQueue( - queue.getQueueName()); - cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueueName()); + queue.getQueuePath()); + cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueuePath()); } } @@ -504,7 +504,7 @@ protected ApplicationId submitApp(MockRM rm, CSQueue parentQueue, // check preconditions List appsInParentQueue = - capacityScheduler.getAppsInQueue(parentQueue.getQueueName()); + capacityScheduler.getAppsInQueue(parentQueue.getQueuePath()); assertEquals(expectedNumAppsInParentQueue, appsInParentQueue.size()); List appsInLeafQueue = @@ -812,8 +812,8 @@ private void validateQueueEntitlementChanges(AutoCreatedLeafQueue leafQueue, boolean found = false; for (QueueManagementChange entitlementChange : queueEntitlementChanges) { - if (leafQueue.getQueueName().equals( - entitlementChange.getQueue().getQueueName())) { + if (leafQueue.getQueuePath().equals( + entitlementChange.getQueue().getQueuePath())) { AutoCreatedLeafQueueConfig updatedQueueTemplate = entitlementChange.getUpdatedQueueTemplate(); @@ -834,7 +834,7 @@ private void validateQueueEntitlementChanges(AutoCreatedLeafQueue leafQueue, if (!found) { fail( "Could not find the specified leaf queue in entitlement changes : " - + leafQueue.getQueueName()); + + leafQueue.getQueuePath()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 7dde06c2da5..a98abb9a004 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -259,7 +259,7 @@ public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception { expectedAbsChildQueueCapacity = populateExpectedAbsCapacityByLabelForParentQueue(1); - validateInitialQueueEntitlement(parentQueue, leafQueue.getQueueName(), + validateInitialQueueEntitlement(parentQueue, leafQueue.getQueuePath(), expectedAbsChildQueueCapacity, accessibleNodeLabelsOnC); } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java index 1b6dd624f7f..e05c8e4ff5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java @@ -156,7 +156,7 @@ public void testSimplePreemption() throws Exception { PreemptionManager pm = cs.getPreemptionManager(); Map killableContainers = - waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 1); + waitKillableContainersSize(pm, "root.a", RMNodeLabelsManager.NO_LABEL, 1); Assert.assertEquals(1, killableContainers.size()); Assert.assertEquals(killableContainers.entrySet().iterator().next().getKey() .getApplicationAttemptId(), am1.getApplicationAttemptId()); @@ -267,7 +267,7 @@ public void testPreemptionConsidersNodeLocalityDelay() PreemptionManager pm = cs.getPreemptionManager(); Map killableContainers = - waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 1); + waitKillableContainersSize(pm, "root.a", RMNodeLabelsManager.NO_LABEL, 1); Assert.assertEquals(killableContainers.entrySet().iterator().next().getKey() .getApplicationAttemptId(), am1.getApplicationAttemptId()); @@ -379,7 +379,7 @@ public void testPreemptionConsidersHardNodeLocality() PreemptionManager pm = cs.getPreemptionManager(); Map killableContainers = - waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 1); + waitKillableContainersSize(pm, "root.a", RMNodeLabelsManager.NO_LABEL, 1); Assert.assertEquals(killableContainers.entrySet().iterator().next().getKey() .getApplicationAttemptId(), am1.getApplicationAttemptId()); @@ -484,7 +484,7 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() editPolicy.editSchedule(); PreemptionManager pm = cs.getPreemptionManager(); - waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 1); + waitKillableContainersSize(pm, "root.a", RMNodeLabelsManager.NO_LABEL, 1); // Check killable containers and to-be-preempted containers in edit policy Assert.assertEquals(0, editPolicy.getToPreemptContainers().size()); @@ -513,7 +513,7 @@ public void testPreemptionPolicyShouldRespectAlreadyMarkedKillableContainers() // Check if previous killable containers included by new killable containers Map killableContainers = - waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 2); + waitKillableContainersSize(pm, "root.a", RMNodeLabelsManager.NO_LABEL, 2); Assert.assertTrue( Sets.difference(previousKillableContainers, killableContainers.keySet()) .isEmpty()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index b0cf8bb76de..5ca6aceb92e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -492,7 +492,7 @@ public RMNodeLabelsManager createNodeLabelManager() { } catch (Exception e) { Assert.assertTrue("Yarn Exception should be thrown", e instanceof YarnException); - Assert.assertEquals("Specified queue=a2 can't satisfy " + Assert.assertEquals("Specified queue=root.a.a2 can't satisfy " + "following apps label expressions =[x] accessible " + "node labels =[y]", e.getMessage()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index b2e71cf1f3c..c58cf5480b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -163,7 +163,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes( when(apps[i].getCurrentAppAttempt()).thenReturn(attempts[i]); rm.getRMContext().getRMApps().put(appids[i], apps[i]); - String queueName = lqs[i % activeQueues].getQueueName(); + String queueName = lqs[i % activeQueues].getQueuePath(); addAppEvent = new AppAddedSchedulerEvent(appids[i], queueName, "user1"); cs.handle(addAppEvent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index c2fea87c621..d9a967859d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -357,9 +357,9 @@ private void testNestedUserQueueWithDynamicParentQueue( assertEquals("Queue", user, ctx.getQueue()); if (primary) { - assertEquals("Primary Group", user + "group", ctx.getParentQueue()); + assertEquals("Primary Group", "root." + user + "group", ctx.getParentQueue()); } else { - assertEquals("Secondary Group", user + "subgroup1", + assertEquals("Secondary Group", "root." + user + "subgroup1", ctx.getParentQueue()); } } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index 95a0cc260e7..146f09febd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -847,7 +847,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { Assert.assertNotNull("Should reserve on nm-" + i, cs.getNode(rmNodes[i].getNodeID()).getReservedContainer()); Assert.assertEquals(cs.getNode(rmNodes[i].getNodeID()) - .getReservedContainer().getQueueName(), queues[0]); + .getReservedContainer().getQueueName(), cs.normalizeQueueName(queues[0])); } // Submit app3 to queue-b and asks for a 0.5G container for AM (on n2) @@ -877,7 +877,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { Assert.assertNotNull("Should reserve on nm-" + i, cs.getNode(rmNodes[i].getNodeID()).getReservedContainer()); Assert.assertEquals(cs.getNode(rmNodes[i].getNodeID()) - .getReservedContainer().getQueueName(), queues[1]); + .getReservedContainer().getQueueName(), cs.normalizeQueueName(queues[1])); } // Sleep the timeout interval, we should be able to see 1 container selected diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index d568220dc68..5cef5839410 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -131,7 +131,7 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable { try { throw new Exception(); } catch (Exception e) { - LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() + + LOG.info("FOOBAR q.assignContainers q=" + queue.getQueuePath() + " alloc=" + allocation + " node=" + node.getNodeName()); } final Resource allocatedResource = Resources.createResource(allocation); @@ -222,8 +222,8 @@ private void setupSortedQueues(CapacitySchedulerConfiguration conf) { public void testSortedQueues() throws Exception { // Setup queue configs setupSortedQueues(csConf); - Map queues = new HashMap(); - CSQueue root = + CSQueueStore queues = new CSQueueStore(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 03c1d604ab2..50cfbcea0c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -129,7 +129,7 @@ CapacitySchedulerContext csContext; CSQueue root; - Map queues; + CSQueueStore queues; final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -158,7 +158,7 @@ private void setUpWithNodeLabels() throws Exception { private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) throws Exception { CapacityScheduler spyCs = new CapacityScheduler(); - queues = new HashMap(); + queues = new CSQueueStore(); cs = spy(spyCs); rmContext = TestUtils.getMockRMContext(); spyRMContext = spy(rmContext); @@ -469,7 +469,7 @@ public void testAppAttemptMetrics() throws Exception { AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(), - a.getQueueName(), user_0); + a.getQueuePath(), user_0); cs.handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId_0, false); @@ -2310,7 +2310,7 @@ public void testRackLocalityDelayScheduling() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); @@ -2746,7 +2746,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -2773,7 +2773,7 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -3120,7 +3120,7 @@ public void testLocalityConstraints() throws Exception { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); final String newRootName = "root" + System.currentTimeMillis(); setupQueueConfiguration(csConf, newRootName, false); @@ -3149,7 +3149,7 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); @@ -4215,9 +4215,9 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() assertEquals(2, leafQueue.getMaxApplicationsPerUser()); //check queue configs - conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueueName(), + conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueuePath(), NO_LABEL, 10); - conf.setMaximumCapacity(leafQueue.getQueueName(), 10); + conf.setMaximumCapacity(leafQueue.getQueuePath(), 10); assertEquals(0.1, leafQueue.getMaxAMResourcePerQueuePercent(), EPSILON); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 4ef9f7a5f7e..89f1964778a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -187,7 +187,7 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable { try { throw new Exception(); } catch (Exception e) { - LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() + + LOG.info("FOOBAR q.assignContainers q=" + queue.getQueuePath() + " alloc=" + allocation + " node=" + node.getNodeName()); } final Resource allocatedResource = Resources.createResource(allocation); @@ -251,8 +251,8 @@ private void verifyQueueMetrics(CSQueue queue, public void testSingleLevelQueues() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -375,7 +375,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + "b"; csConf.setCapacity(Q_B, 70.5F); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); boolean exceptionOccurred = false; try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, @@ -496,8 +496,8 @@ public void testMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -662,8 +662,8 @@ public void testQueueCapacitySettingChildZero() throws Exception { csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -680,7 +680,7 @@ public void testQueueCapacitySettingParentZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -701,7 +701,7 @@ public void testQueueCapacityZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -717,7 +717,7 @@ public void testOffSwitchScheduling() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -799,7 +799,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); //B3 - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -901,7 +901,7 @@ public void testQueueAcl() throws Exception { final String Q_C11= Q_C + "." + C1 + "." + C11; csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -929,27 +929,27 @@ public void testQueueAcl() throws Exception { // c has no SA, but QA assertTrue(c.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c")); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "root.c")); assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); - assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c")); + assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "root.c")); //Queue c1 has QA, no SA (gotten perm from parent) assertTrue(c1.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c1")); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "root.c.c1")); assertFalse(c1.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); - assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c1")); + assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "root.c.c1")); //Queue c11 has permissions from parent queue and SA assertTrue(c11.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c11")); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "root.c.c1.c11")); assertTrue(c11.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c11")); + assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "root.c.c1.c11")); //Queue c111 has SA and AQ, both from parent assertTrue(c111.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c111")); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "root.c.c1.c11.c111")); assertTrue(c111.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); - assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c111")); + assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "root.c.c1.c11.c111")); reset(c); } @@ -960,7 +960,7 @@ public void testAbsoluteResourceWithChangeInClusterResource() // Setup queue configs setupSingleLevelQueuesWithAbsoluteResource(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 2a8af9281ef..093cea59ded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -1146,7 +1146,7 @@ public void testQueueOrderingPolicyUpdatedAfterReinitialize() private void checkEqualsToQueueSet(List queues, String[] queueNames) { Set existedQueues = new HashSet<>(); for (CSQueue q : queues) { - existedQueues.add(q.getQueueName()); + existedQueues.add(q.getQueueShortName()); } for (String q : queueNames) { Assert.assertTrue(existedQueues.remove(q)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java index 45aaedd8fec..9997335f208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java @@ -111,8 +111,8 @@ public void testQueueState() throws IOException { Assert.fail("Should throw an Exception."); } catch (Exception ex) { Assert.assertTrue(ex.getCause().getMessage().contains( - "The parent queue:q1 cannot be STOPPED as the child" + - " queue:q2 is in RUNNING state.")); + "The parent queue:root.q1 cannot be STOPPED as the child" + + " queue:root.q1.q2 is in RUNNING state.")); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index da0e9c9e6fc..93132a95115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -95,8 +95,8 @@ private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); CSQueue root; - Map queues = new HashMap(); - Map oldQueues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); + CSQueueStore oldQueues = new CSQueueStore(); final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -546,7 +546,7 @@ public void testReservationNoContinueLook() throws Exception { // Test that with reservations-continue-look-all-nodes feature off // we don't unreserve and show we could get stuck - queues = new HashMap(); + queues = new CSQueueStore(); // test that the deadlock occurs when turned off CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, @@ -1188,7 +1188,7 @@ public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, csConf.setBoolean( CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java index ca9a84b304d..519b61156eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java @@ -45,7 +45,7 @@ List list = new ArrayList<>(); for (int i = 0; i < queueNames.length; i++) { CSQueue q = mock(CSQueue.class); - when(q.getQueueName()).thenReturn(queueNames[i]); + when(q.getQueuePath()).thenReturn(queueNames[i]); QueueCapacities qc = new QueueCapacities(false); qc.setAbsoluteCapacity(partition, absCapacities[i]); @@ -68,7 +68,7 @@ private void verifyOrder(QueueOrderingPolicy orderingPolicy, String partition, int i = 0; while (iter.hasNext()) { CSQueue q = iter.next(); - Assert.assertEquals(expectedOrder[i], q.getQueueName()); + Assert.assertEquals(expectedOrder[i], q.getQueuePath()); i++; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index b5e30d959c8..7e240bd4596 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -100,6 +100,7 @@ private static final String LABEL_LX = "Lx"; private static final ImmutableSet CLUSTER_LABELS = ImmutableSet.of(LABEL_LX, LABEL_LY, DEFAULT_PARTITION); + private static final String DOT = "."; private static MockRM rm; static private CapacitySchedulerConfiguration csConf; static private YarnConfiguration conf; @@ -310,7 +311,7 @@ public void testPartitionInSchedulerActivities() throws Exception { verifyNumberOfAllocations(schedulerActivitiesJson, 1); // verify queue Qb Predicate findQueueBPred = - (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(QUEUE_B); + (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(CapacitySchedulerConfiguration.ROOT + DOT + QUEUE_B); List queueBObj = ActivitiesTestUtils.findInAllocations( getFirstSubNodeFromJson(schedulerActivitiesJson, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS), findQueueBPred); @@ -322,7 +323,7 @@ public void testPartitionInSchedulerActivities() throws Exception { queueBObj.get(0).optString(FN_ACT_DIAGNOSTIC)); // verify queue Qa Predicate findQueueAPred = - (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(QUEUE_A); + (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(CapacitySchedulerConfiguration.ROOT + DOT + QUEUE_A); List queueAObj = ActivitiesTestUtils.findInAllocations( getFirstSubNodeFromJson(schedulerActivitiesJson, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS), findQueueAPred); @@ -334,7 +335,7 @@ public void testPartitionInSchedulerActivities() throws Exception { queueAObj.get(0).optString(FN_ACT_DIAGNOSTIC)); // verify queue Qc Predicate findQueueCPred = - (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(QUEUE_C); + (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals(CapacitySchedulerConfiguration.ROOT + DOT + QUEUE_C); List queueCObj = ActivitiesTestUtils.findInAllocations( getFirstSubNodeFromJson(schedulerActivitiesJson, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS), findQueueCPred); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 1627460800c..1dd80204829 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -154,7 +154,7 @@ public void testAssignMultipleContainersPerNodeHeartbeat() verifyStateOfAllocations(allocation, FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED"); verifyQueueOrder(allocation, - "root-a-b-b2-b3-b1"); + "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1"); } finally { rm.stop(); } @@ -378,7 +378,7 @@ public void testReserveNewContainer() throws Exception { JSONObject allocations = getFirstSubNodeFromJson(json, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS); verifyQueueOrder(allocations, - "root-a-b-b3-b1"); + "root-root.a-root.b-root.b.b3-root.b.b1"); verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE, "RESERVED"); @@ -407,7 +407,7 @@ public void testReserveNewContainer() throws Exception { JSONObject allocation = getFirstSubNodeFromJson(json, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS); - verifyQueueOrder(allocation, "b1"); + verifyQueueOrder(allocation, "root.b.b1"); verifyStateOfAllocations(allocation, FN_ACT_FINAL_ALLOCATION_STATE, "RESERVED"); @@ -444,7 +444,7 @@ public void testReserveNewContainer() throws Exception { allocations = getFirstSubNodeFromJson(json, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS); - verifyQueueOrder(allocations, "b1"); + verifyQueueOrder(allocations, "root.b.b1"); verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED_FROM_RESERVED"); } finally { @@ -504,7 +504,7 @@ public void testActivityJSON() throws Exception { // Increase number of nodes to 6 since request node has been added verifyNumberOfNodes(allocation, 6); - verifyQueueOrder(allocation, "root-b-b1"); + verifyQueueOrder(allocation, "root-root.b-root.b.b1"); } finally { rm.stop(); } @@ -1562,7 +1562,7 @@ public void testQueueSkippedBecauseOfHeadroom() throws Exception { verifyNumberOfAllocations(schedulerActivitiesJson, 1); // verify at queue level Predicate findA1AQueuePred = - (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals("a1a"); + (obj) -> obj.optString(FN_SCHEDULER_ACT_NAME).equals("root.a.a1.a1a"); List a1aQueueObj = ActivitiesTestUtils.findInAllocations( getFirstSubNodeFromJson(schedulerActivitiesJson, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS), findA1AQueuePred); -- 2.20.1 (Apple Git-117)