From b04be2c35702557479a7c6ceea1593c017789d2f Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Tue, 21 Jan 2020 17:22:03 +0100 Subject: [PATCH] YARN-9879 Allow multiple leaf queues with the same name in CS --- .../AbstractAutoCreatedLeafQueue.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 42 +++--- .../capacity/AbstractManagedParentQueue.java | 2 +- .../scheduler/capacity/CSQueue.java | 8 +- .../scheduler/capacity/CSQueueStore.java | 127 ++++++++++++++++++ .../scheduler/capacity/CapacityScheduler.java | 22 +-- .../CapacitySchedulerQueueManager.java | 81 +++++------ .../scheduler/capacity/LeafQueue.java | 32 ++--- .../capacity/ManagedParentQueue.java | 8 +- .../scheduler/capacity/ParentQueue.java | 50 +++---- .../capacity/QueueManagementChange.java | 2 +- .../QueueManagementDynamicEditPolicy.java | 4 +- .../scheduler/capacity/UsersManager.java | 2 +- ...uaranteedOrZeroCapacityOverTimePolicy.java | 22 +-- .../capacity/TestApplicationLimits.java | 14 +- .../TestApplicationLimitsByPartition.java | 2 +- .../scheduler/capacity/TestCSQueueStore.java | 13 ++ .../capacity/TestChildQueueOrder.java | 4 +- .../scheduler/capacity/TestLeafQueue.java | 14 +- .../scheduler/capacity/TestParentQueue.java | 26 ++-- .../scheduler/capacity/TestReservations.java | 8 +- 21 files changed, 317 insertions(+), 168 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 9e5bdb0bc0b..bed2aa5b06d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -93,7 +93,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement) // this might be revised later setMaxCapacity(nodeLabel, entitlement.getMaxCapacity()); LOG.debug("successfully changed to {} for queue {}", capacity, this - .getQueueName()); + .getQueuePath()); //update queue used capacity etc CSQueueUtils.updateQueueStatistics(resourceCalculator, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 3ae22ad0ebd..5df0334a723 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -109,6 +109,8 @@ // Track resource usage-by-label like used-resource/pending-resource, etc. volatile ResourceUsage queueUsage; + + private final boolean fullPathQueueNamingPolicy = false; // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. @@ -193,7 +195,7 @@ protected void setupConfigurableCapacities( public String getQueuePath() { return queuePath; } - + @Override public float getCapacity() { return queueCapacities.getCapacity(); @@ -243,8 +245,16 @@ public CSQueueMetrics getMetrics() { return metrics; } + @Override + public String getQueueShortName() { + return queueName; + } + @Override public String getQueueName() { + if (fullPathQueueNamingPolicy) { + return queuePath; + } return queueName; } @@ -282,11 +292,11 @@ void setMaxCapacity(float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueueName(), + CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueueName(), + CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -304,11 +314,11 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueueName(), + CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(nodeLabel), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueueName(), + CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -503,7 +513,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, queuePath, resourceTypes); LOG.debug("capacityConfigType is '{}' for queue {}", - capacityConfigType, getQueueName()); + capacityConfigType, getQueuePath()); if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { this.capacityConfigType = (!minResource.equals(Resources.none()) @@ -511,7 +521,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, ? CapacityConfigType.ABSOLUTE_RESOURCE : CapacityConfigType.PERCENTAGE; LOG.debug("capacityConfigType is updated as '{}' for queue {}", - capacityConfigType, getQueueName()); + capacityConfigType, getQueuePath()); } validateAbsoluteVsPercentageCapacityConfig(minResource); @@ -522,7 +532,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, resourceCalculator, clusterResource, minResource, maxResource)) { throw new IllegalArgumentException("Min resource configuration " + minResource + " is greater than its max value:" + maxResource - + " in queue:" + getQueueName()); + + " in queue:" + getQueuePath()); } // If parent's max resource is lesser to a specific child's max @@ -536,7 +546,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, maxResource, parentMaxRes)) { throw new IllegalArgumentException("Max resource configuration " + maxResource + " is greater than parents max value:" - + parentMaxRes + " in queue:" + getQueueName()); + + parentMaxRes + " in queue:" + getQueuePath()); } // If child's max resource is not set, but its parent max resource is @@ -549,7 +559,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, } LOG.debug("Updating absolute resource configuration for queue:{} as" - + " minResource={} and maxResource={}", getQueueName(), minResource, + + " minResource={} and maxResource={}", getQueuePath(), minResource, maxResource); queueResourceQuotas.setConfiguredMinResource(label, minResource); @@ -566,7 +576,7 @@ private void validateAbsoluteVsPercentageCapacityConfig( if (!queueName.equals("root") && !this.capacityConfigType.equals(localType)) { - throw new IllegalArgumentException("Queue '" + getQueueName() + throw new IllegalArgumentException("Queue '" + getQueuePath() + "' should use either percentage based capacity" + " configuration or absolute resource."); } @@ -628,7 +638,7 @@ private void initializeQueueState(QueueState previousState, } else if (configuredState == QueueState.RUNNING && parentState != QueueState.RUNNING) { throw new IllegalArgumentException( - "The parent queue:" + parent.getQueueName() + "The parent queue:" + parent.getQueuePath() + " cannot be STOPPED as the child queue:" + queueName + " is in RUNNING state."); } else { @@ -982,7 +992,7 @@ boolean canAssignToThisQueue(Resource clusterResource, if (Resources.lessThan(resourceCalculator, clusterResource, newTotalWithoutReservedResource, currentLimitResource)) { if (LOG.isDebugEnabled()) { - LOG.debug("try to use reserved: " + getQueueName() + LOG.debug("try to use reserved: " + getQueuePath() + " usedResources: " + queueUsage.getUsed() + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved @@ -996,7 +1006,7 @@ boolean canAssignToThisQueue(Resource clusterResource, // Can not assign to this queue if (LOG.isDebugEnabled()) { - LOG.debug("Failed to assign to queue: " + getQueueName() + LOG.debug("Failed to assign to queue: " + getQueuePath() + " nodePatrition: " + nodePartition + ", usedResources: " + queueUsage.getUsed(nodePartition) + ", clusterResources: " + clusterResource @@ -1007,7 +1017,7 @@ boolean canAssignToThisQueue(Resource clusterResource, return false; } if (LOG.isDebugEnabled()) { - LOG.debug("Check assign to queue: " + getQueueName() + LOG.debug("Check assign to queue: " + getQueuePath() + " nodePartition: " + nodePartition + ", usedResources: " + queueUsage.getUsed(nodePartition) + ", clusterResources: " + clusterResource @@ -1262,7 +1272,7 @@ public void activeQueue() throws YarnException { if (parent == null || parent.getState() == QueueState.RUNNING) { updateQueueState(QueueState.RUNNING); } else { - throw new YarnException("The parent Queue:" + parent.getQueueName() + throw new YarnException("The parent Queue:" + parent.getQueuePath() + " is not running. Please activate the parent queue first"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 7626c66d555..7bdc311aea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -106,7 +106,7 @@ public void removeChildQueue(CSQueue childQueue) CSQueue cs = qiter.next(); if (cs.equals(childQueue)) { qiter.remove(); - LOG.debug("Removed child queue: {}", cs.getQueueName()); + LOG.debug("Removed child queue: {}", cs.getQueuePath()); } } } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index d507e53543c..da6c5ea1fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -73,11 +73,17 @@ public void setParent(CSQueue newParentQueue); /** - * Get the queue name. + * Get the queue's internal reference name. * @return the queue name */ public String getQueueName(); + /** + * Get the queue's legacy name + * @return the queue name + */ + public String getQueueShortName(); + /** * Get the full name of the queue, including the heirarchy. * @return the full name of the queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java new file mode 100644 index 00000000000..b6970d87448 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java @@ -0,0 +1,127 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import com.google.common.collect.ImmutableMap; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class CSQueueStore { + private final Map fullNameQueues = new ConcurrentHashMap<>(); + private final Map shortNameToFullName = new HashMap<>(); + private final Map> ambiguousShortNames = new HashMap<>(); + + public Map getFullNameQueues() { + return ImmutableMap.copyOf(fullNameQueues); + } + + //we must synchronize here because we need to maintain multiple maps to be + //in sync, and concurrent hashMap does not help with that + private synchronized void addShortNameMapping(String shortName, String fullName) { + if (ambiguousShortNames.containsKey(shortName)) { + //if the short name is already ambiguous we just add it to the list + ambiguousShortNames.get(shortName).add(fullName); + } else if (shortNameToFullName.containsKey(shortName) + && !fullName.equals(shortNameToFullName.get(shortName))) { + + //the short name became ambiguous creating the ambiguous list + Set ambiguousQueues = new HashSet<>(); + ambiguousQueues.add(fullName); + ambiguousQueues.add(shortNameToFullName.get(shortName)); + ambiguousShortNames.put(shortName, ambiguousQueues); + + //removing the short name from queue name map, since there is no + //disambiguous mapping + shortNameToFullName.remove(shortName); + } else { + //there is no issue with the queue's short name we just add it to the map + shortNameToFullName.put(shortName, fullName); + } + } + + public void add(CSQueue queue) { + String fullName = queue.getQueuePath(); + String shortName = queue.getQueueShortName(); + + fullNameQueues.put(fullName, queue); + if (queue instanceof LeafQueue) { + addShortNameMapping(shortName, fullName); + } + } + + public synchronized void remove(CSQueue queue) { + //if no queue is specified, we can consider it already removed, also consistent + //with hashmap behaviour, so no new issues will be caused by it + if (queue == null) { + return; + } + + String fullName = queue.getQueuePath(); + String shortName = queue.getQueueShortName(); + + //removing from the full and short name maps as well + fullNameQueues.remove(fullName); + if (shortNameToFullName.containsKey(shortName)) { + shortNameToFullName.remove(shortName); + } + + if (ambiguousShortNames.containsKey(shortName) && queue instanceof LeafQueue) { + //if the removed queued was an ambiguous one, we check if removing it + //resolves the issue. + Set conflictingQueues = ambiguousShortNames.get(shortName); + conflictingQueues.remove(fullName); + if (conflictingQueues.size() == 1) { + //if we have only 1 conflictingQueue, it is not a conflict, resolving + ambiguousShortNames.remove(shortName); + shortNameToFullName.put(shortName, conflictingQueues.iterator().next()); + } + } + + } + + + public synchronized void remove(String name) { + CSQueue queue = get(name); + if (queue != null) remove(queue); + } + + public int getQueueCount() { + return fullNameQueues.size(); + } + + public CSQueue getByFullName(String name) { + return fullNameQueues.getOrDefault(name, null); + } + + public CSQueue getByShortName(String name) { + String mappedName = shortNameToFullName.getOrDefault(name, null); + if (mappedName != null) { + return fullNameQueues.get(mappedName); + } + + return null; + } + + public CSQueue get(String name) { + CSQueue ret = getByFullName(name); + + if (ret == null) { + ret = getByShortName(name); + } + + return ret; + } + + public synchronized void clear() { + fullNameQueues.clear(); + shortNameToFullName.clear(); + ambiguousShortNames.clear(); + } + + public Collection getQueues() { + return fullNameQueues.values(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index fcc05602871..2b739dfb91c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1073,7 +1073,7 @@ private void addApplicationAttempt( queue.submitApplicationAttempt(attempt, application.getUser()); LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user " + application.getUser() + " in queue " - + queue.getQueueName()); + + queue.getQueuePath()); if (isAttemptRecovering) { LOG.debug("{} is recovering. Skipping notifying ATTEMPT_ADDED", applicationAttemptId); @@ -1102,7 +1102,7 @@ private void doneApplication(ApplicationId applicationId, CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue - .getQueueName()); + .getQueuePath()); } else{ queue.finishApplication(applicationId, application.getUser()); } @@ -1541,7 +1541,7 @@ private CSAssignment allocateContainerOnSingleNode( LOG.error("Trying to schedule on a removed node, please double check, " + "nodeId=" + node.getNodeID()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.INIT_CHECK_SINGLE_NODE_REMOVED); ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, node); @@ -1570,7 +1570,7 @@ private CSAssignment allocateContainerOnSingleNode( LOG.debug("This node " + node.getNodeID() + " doesn't have sufficient " + "available or preemptible resource for minimum allocation"); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant. INIT_CHECK_SINGLE_NODE_RESOURCE_INSUFFICIENT); ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -1619,14 +1619,14 @@ private void allocateFromReservedContainer(FiCaSchedulerNode node, schedulerHealth.updateSchedulerFulfilledReservationCounts(1); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), + queue.getParent().getQueuePath(), queue.getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.ALLOCATED_FROM_RESERVED); } else if (assignment.getAssignmentInformation().getNumReservations() > 0) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), + queue.getParent().getQueuePath(), queue.getQueuePath(), ActivityState.RE_RESERVED, ActivityDiagnosticConstant.EMPTY); ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.RESERVED); @@ -1718,7 +1718,7 @@ private CSAssignment allocateContainersOnMultiNodes( LOG.debug("This partition '{}' doesn't have available or " + "killable resource", candidates.getPartition()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, null, - "", getRootQueue().getQueueName(), ActivityState.REJECTED, + "", getRootQueue().getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant. INIT_CHECK_PARTITION_RESOURCE_INSUFFICIENT); ActivitiesLogger.NODE @@ -1932,10 +1932,10 @@ public void handle(SchedulerEvent event) { .validateAndApplyQueueManagementChanges(queueManagementChanges); } catch (SchedulerDynamicEditException sde) { LOG.error("Queue Management Change event cannot be applied for " - + "parent queue : " + parentQueue.getQueueName(), sde); + + "parent queue : " + parentQueue.getQueuePath(), sde); } catch (IOException ioe) { LOG.error("Queue Management Change event cannot be applied for " - + "parent queue : " + parentQueue.getQueueName(), ioe); + + "parent queue : " + parentQueue.getQueuePath(), ioe); } } break; @@ -2396,7 +2396,7 @@ public void addQueue(Queue queue) AbstractManagedParentQueue parent = (AbstractManagedParentQueue) newQueue.getParent(); - String queuename = newQueue.getQueueName(); + String queuename = newQueue.getQueuePath(); parent.addChildQueue(newQueue); this.queueManager.addQueue(queuename, newQueue); @@ -2559,7 +2559,7 @@ private void checkQueuePartition(FiCaSchedulerApp app, LeafQueue dest) } if (nonAccessiblelabels.size() > 0) { throw new YarnException( - "Specified queue=" + dest.getQueueName() + " can't satisfy following " + "Specified queue=" + dest.getQueuePath() + " can't satisfy following " + "apps label expressions =" + nonAccessiblelabels + " accessible node labels =" + targetqueuelabels); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index d9b3ebd2eda..f57dd98eeea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +90,7 @@ public CSQueue hook(CSQueue queue) { private static final QueueHook NOOP = new QueueHook(); private CapacitySchedulerContext csContext; private final YarnAuthorizationProvider authorizer; - private final Map queues = new ConcurrentHashMap<>(); + private final CSQueueStore queues = new CSQueueStore(); private CSQueue root; private final RMNodeLabelsManager labelManager; private AppPriorityACLsManager appPriorityACLManager; @@ -121,7 +120,7 @@ public CSQueue getRootQueue() { @Override public Map getQueues() { - return queues; + return queues.getFullNameQueues(); } @Override @@ -131,7 +130,7 @@ public void removeQueue(String queueName) { @Override public void addQueue(String queueName, CSQueue queue) { - this.queues.put(queueName, queue); + this.queues.add(queue); } @Override @@ -167,7 +166,7 @@ public void initializeQueues(CapacitySchedulerConfiguration conf) public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues - Map newQueues = new HashMap<>(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); @@ -212,8 +211,9 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) static CSQueue parseQueue( CapacitySchedulerContext csContext, CapacitySchedulerConfiguration conf, - CSQueue parent, String queueName, Map queues, - Map oldQueues, + CSQueue parent, String queueName, + CSQueueStore queues, + CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; String fullQueueName = (parent == null) ? @@ -232,7 +232,7 @@ static CSQueue parseQueue( // system if (isReservableQueue) { queue = new PlanQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); //initializing the "internal" default queue, for SLS compatibility String defReservationId = @@ -248,31 +248,31 @@ static CSQueue parseQueue( } childQueues.add(resQueue); ((PlanQueue) queue).setChildQueues(childQueues); - queues.put(defReservationId, resQueue); + queues.add(resQueue); } else if (isAutoCreateEnabled) { queue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ queue = new LeafQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); // Used only for unit tests queue = hook.hook(queue); } } else{ if (isReservableQueue) { throw new IllegalStateException( - "Only Leaf Queues can be reservable for " + queueName); + "Only Leaf Queues can be reservable for " + fullQueueName); } ParentQueue parentQueue; if (isAutoCreateEnabled) { parentQueue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ parentQueue = new ParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } // Used only for unit tests @@ -288,14 +288,14 @@ static CSQueue parseQueue( } - if (queue instanceof LeafQueue && queues.containsKey(queueName) && queues - .get(queueName) instanceof LeafQueue) { - throw new IOException("Two leaf queues were named " + queueName + if (queue instanceof LeafQueue && queues.get(fullQueueName) != null && queues + .get(fullQueueName) instanceof LeafQueue) { + throw new IOException("Two leaf queues were named " + fullQueueName + ". Leaf queue names must be distinct"); } - queues.put(queueName, queue); + queues.add(queue); - LOG.info("Initialized queue: " + queue); + LOG.info("Initialized queue: " + fullQueueName); return queue; } @@ -308,15 +308,14 @@ static CSQueue parseQueue( * @param queues existing queues * @param newQueues new queues */ - private void validateQueueHierarchy(Map queues, - Map newQueues, CapacitySchedulerConfiguration newConf) + private void validateQueueHierarchy(CSQueueStore queues, + CSQueueStore newQueues, CapacitySchedulerConfiguration newConf) throws IOException { // check that all static queues are included in the newQueues list - for (Map.Entry e : queues.entrySet()) { - if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue() + for (CSQueue oldQueue : queues.getQueues()) { + if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(oldQueue .getClass()))) { - String queueName = e.getKey(); - CSQueue oldQueue = e.getValue(); + String queueName = oldQueue.getQueuePath(); CSQueue newQueue = newQueues.get(queueName); if (null == newQueue) { // old queue doesn't exist in the new XML @@ -391,27 +390,21 @@ private void validateQueueHierarchy(Map queues, * @param existingQueues, the existing queues * @param newQueues the new queues based on new XML */ - private void updateQueues(Map existingQueues, - Map newQueues) { + private void updateQueues(CSQueueStore existingQueues, + CSQueueStore newQueues) { CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - for (Map.Entry e : newQueues.entrySet()) { - String queueName = e.getKey(); - CSQueue queue = e.getValue(); - if (!existingQueues.containsKey(queueName)) { - existingQueues.put(queueName, queue); + for (CSQueue queue : newQueues.getQueues()) { + if (existingQueues.get(queue.getQueuePath()) == null) { + existingQueues.add(queue); } } - for (Iterator> itr = existingQueues.entrySet() - .iterator(); itr.hasNext();) { - Map.Entry e = itr.next(); - String queueName = e.getKey(); - CSQueue existingQueue = e.getValue(); - - if (!newQueues.containsKey(queueName) && !( - existingQueue instanceof AutoCreatedLeafQueue && conf + + for (CSQueue queue: existingQueues.getQueues()) { + if (existingQueues.get(queue.getQueuePath()) == null && !( + queue instanceof AutoCreatedLeafQueue && conf .isAutoCreateChildQueueEnabled( - existingQueue.getParent().getQueuePath()))) { - itr.remove(); + queue.getParent().getQueuePath()))) { + existingQueues.remove(queue); } } } @@ -424,10 +417,10 @@ private void updateQueues(Map existingQueues, * @throws IOException if fails to set queue acls */ public static void setQueueAcls(YarnAuthorizationProvider authorizer, - AppPriorityACLsManager appPriorityACLManager, Map queues) + AppPriorityACLsManager appPriorityACLManager, CSQueueStore queues) throws IOException { List permissions = new ArrayList<>(); - for (CSQueue queue : queues.values()) { + for (CSQueue queue : queues.getQueues()) { AbstractCSQueue csQueue = (AbstractCSQueue) queue; permissions.add( new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1028a7d0b2e..47383affc8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -217,7 +217,7 @@ protected void setupQueueConfigs(Resource clusterResource, if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, this.defaultLabelExpression, null)) { throw new IOException( - "Invalid default label expression of " + " queue=" + getQueueName() + "Invalid default label expression of " + " queue=" + getQueuePath() + " doesn't have permission to access all labels " + "in default label expression. labelExpression of resource request=" + (this.defaultLabelExpression == null ? @@ -781,7 +781,7 @@ public Resource calculateAndGetAMResourceLimitPerPartition( queueUsage.setAMLimit(nodePartition, amResouceLimit); LOG.debug("Queue: {}, node label : {}, queue partition resource : {}," + " queue current limit : {}, queue partition usable resource : {}," - + " amResourceLimit : {}", getQueueName(), nodePartition, + + " amResourceLimit : {}", getQueuePath(), nodePartition, queuePartitionResource, queueCurrentLimit, queuePartitionUsableResource, amResouceLimit); return amResouceLimit; @@ -898,7 +898,7 @@ protected void activateApplications() { application.getUser(), userAMLimit); fsApp.remove(); LOG.info("Application " + applicationId + " from user: " + application - .getUser() + " activated in queue: " + getQueueName()); + .getUser() + " activated in queue: " + getQueuePath()); } } finally { writeLock.unlock(); @@ -930,7 +930,7 @@ private void addApplicationAttempt(FiCaSchedulerApp application, LOG.info( "Application added -" + " appId: " + application.getApplicationId() + " user: " + application.getUser() + "," + " leaf-queue: " - + getQueueName() + " #user-pending-applications: " + user + + getQueuePath() + " #user-pending-applications: " + user .getPendingApplications() + " #user-active-applications: " + user .getActiveApplications() + " #queue-pending-applications: " + getNumPendingApplications() + " #queue-active-applications: " @@ -991,7 +991,7 @@ private void removeApplicationAttempt( LOG.info( "Application removed -" + " appId: " + application.getApplicationId() - + " user: " + application.getUser() + " queue: " + getQueueName() + + " user: " + application.getUser() + " queue: " + getQueuePath() + " #user-pending-applications: " + user.getPendingApplications() + " #user-active-applications: " + user.getActiveApplications() + " #queue-pending-applications: " + getNumPendingApplications() @@ -1074,7 +1074,7 @@ public CSAssignment assignContainers(Resource clusterResource, if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY && !accessibleToPartition(candidates.getPartition())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); return CSAssignment.NULL_ASSIGNMENT; } @@ -1090,7 +1090,7 @@ public CSAssignment assignContainers(Resource clusterResource, .getPartition()); } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); return CSAssignment.NULL_ASSIGNMENT; } @@ -1117,7 +1117,7 @@ public CSAssignment assignContainers(Resource clusterResource, activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); return CSAssignment.NULL_ASSIGNMENT; @@ -1180,7 +1180,7 @@ public CSAssignment assignContainers(Resource clusterResource, if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), + getParent().getQueuePath(), getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); return assignment; } else if (assignment.getSkippedType() @@ -1192,7 +1192,7 @@ public CSAssignment assignContainers(Resource clusterResource, } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, + getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + " from " + application.getApplicationId()); return assignment; @@ -1200,7 +1200,7 @@ public CSAssignment assignContainers(Resource clusterResource, // If we don't allocate anything, and it is not skipped by application, // we will return to respect FIFO of applications ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); ActivitiesLogger.APP.finishSkippedAppAllocationRecording( activitiesManager, application.getApplicationId(), @@ -1209,7 +1209,7 @@ public CSAssignment assignContainers(Resource clusterResource, } } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, + getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); return CSAssignment.NULL_ASSIGNMENT; @@ -1563,7 +1563,7 @@ protected boolean canAssignToUser(Resource clusterResource, application.getCurrentReservation()), limit)) { if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueueName() + LOG.debug("User " + userName + " in queue " + getQueuePath() + " will exceed limit based on reservations - " + " consumed: " + user.getUsed() + " reserved: " + application .getCurrentReservation() + " limit: " + limit); @@ -1578,7 +1578,7 @@ protected boolean canAssignToUser(Resource clusterResource, } } if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueueName() + LOG.debug("User " + userName + " in queue " + getQueuePath() + " will exceed limit - " + " consumed: " + user .getUsed(nodePartition) + " limit: " + limit); } @@ -1735,7 +1735,7 @@ void allocateResource(Resource clusterResource, userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { - LOG.debug(getQueueName() + " user=" + userName + " used=" + LOG.debug(getQueuePath() + " user=" + userName + " used=" + queueUsage.getUsed(nodePartition) + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed()); @@ -1776,7 +1776,7 @@ void releaseResource(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug( - getQueueName() + " used=" + queueUsage.getUsed() + " numContainers=" + getQueuePath() + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " user=" + userName + " user-resources=" + user.getUsed()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 6c40a23f448..d7607ceb947 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -121,7 +121,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) queueName, super.getCapacity(), super.getMaximumCapacity()); } catch (YarnException ye) { LOG.error("Exception while computing policy changes for leaf queue : " - + getQueueName(), ye); + + getQueuePath(), ye); throw new IOException(ye); } finally { writeLock.unlock(); @@ -196,7 +196,7 @@ public void addChildQueue(CSQueue childQueue) ManagedParentQueue parentQueue = (ManagedParentQueue) childQueue.getParent(); - String leafQueueName = childQueue.getQueueName(); + String leafQueueName = childQueue.getQueuePath(); int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit( parentQueue.getQueuePath()); @@ -320,11 +320,11 @@ public void validateQueueManagementChanges( if (!(AbstractManagedParentQueue.class. isAssignableFrom(childQueue.getParent().getClass()))) { - LOG.error("Queue " + getQueueName() + LOG.error("Queue " + getQueuePath() + " is not an instance of PlanQueue or ManagedParentQueue." + " " + "Ignoring update " + queueManagementChanges); throw new SchedulerDynamicEditException( - "Queue " + getQueueName() + " is not a AutoEnabledParentQueue." + "Queue " + getQueuePath() + " is not a AutoEnabledParentQueue." + " Ignoring update " + queueManagementChanges); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 8d3244722cf..96ec2664747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -182,8 +182,8 @@ void setChildQueues(Collection childQueues) { if (!queueName.equals("root") && getCapacity() != 0f && !queue.getQueueResourceQuotas().getConfiguredMinResource() .equals(Resources.none())) { - throw new IllegalArgumentException("Parent queue '" + getQueueName() - + "' and child queue '" + queue.getQueueName() + throw new IllegalArgumentException("Parent queue '" + getQueuePath() + + "' and child queue '" + queue.getQueuePath() + "' should use either percentage based capacity" + " configuration or absolute resource together."); } @@ -213,8 +213,8 @@ void setChildQueues(Collection childQueues) { // back an exception if (!queueName.equals("root") && !this.capacityConfigType .equals(queue.getCapacityConfigType())) { - throw new IllegalArgumentException("Parent queue '" + getQueueName() - + "' and child queue '" + queue.getQueueName() + throw new IllegalArgumentException("Parent queue '" + getQueuePath() + + "' and child queue '" + queue.getQueuePath() + "' should use either percentage based capacity" + "configuration or absolute resource together for label:" + nodeLabel); @@ -383,7 +383,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } // Re-init existing queues childQueue.reinitialize(newChildQueue, clusterResource); - LOG.info(getQueueName() + ": re-configured queue: " + childQueue); + LOG.info(getQueuePath() + ": re-configured queue: " + childQueue); } else{ // New child queue, do not re-init @@ -394,7 +394,7 @@ public void reinitialize(CSQueue newlyParsedQueue, currentChildQueues.put(newChildQueueName, newChildQueue); LOG.info( - getQueueName() + ": added new child queue: " + newChildQueue); + getQueuePath() + ": added new child queue: " + newChildQueue); } } @@ -499,7 +499,7 @@ private void addApplication(ApplicationId applicationId, LOG.info( "Application added -" + " appId: " + applicationId + " user: " + user - + " leaf-queue of parent: " + getQueueName() + " #applications: " + + " leaf-queue of parent: " + getQueuePath() + " #applications: " + getNumApplications()); } finally { writeLock.unlock(); @@ -526,7 +526,7 @@ private void removeApplication(ApplicationId applicationId, --numApplications; LOG.info("Application removed -" + " appId: " + applicationId + " user: " - + user + " leaf-queue of parent: " + getQueueName() + + user + " leaf-queue of parent: " + getQueuePath() + " #applications: " + getNumApplications()); } finally { writeLock.unlock(); @@ -558,7 +558,7 @@ public CSAssignment assignContainers(Resource clusterResource, } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -585,7 +585,7 @@ public CSAssignment assignContainers(Resource clusterResource, } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.SKIPPED, + getParentName(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -600,7 +600,7 @@ public CSAssignment assignContainers(Resource clusterResource, while (canAssign(clusterResource, node)) { LOG.debug("Trying to assign containers to child-queue of {}", - getQueueName()); + getQueuePath()); // Are we over maximum-capacity for this queue? // This will also consider parent's limits and also continuous reservation @@ -612,7 +612,7 @@ public CSAssignment assignContainers(Resource clusterResource, getMetrics().getReservedVirtualCores()), schedulingMode)) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -640,7 +640,7 @@ public CSAssignment assignContainers(Resource clusterResource, assignedToChild.getResource(), Resources.none())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.ACCEPTED, + getParentName(), getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); boolean isReserved = @@ -678,12 +678,12 @@ public CSAssignment assignContainers(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("assignedContainer reserved=" + isReserved + " queue=" - + getQueueName() + " usedCapacity=" + getUsedCapacity() + + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); LOG.debug( - "ParentQ=" + getQueueName() + " assignedSoFarInThisIteration=" + "ParentQ=" + getQueuePath() + " assignedSoFarInThisIteration=" + assignment.getResource() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity()); @@ -692,7 +692,7 @@ public CSAssignment assignContainers(Resource clusterResource, assignment.setSkippedType(assignedToChild.getSkippedType()); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.SKIPPED, + getParentName(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); if (rootQueue) { ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager, @@ -731,7 +731,7 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) { node.getTotalKillableResources()), minimumAllocation); if (!accept) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParentName(), getQueueName(), ActivityState.REJECTED, + getParentName(), getQueuePath(), ActivityState.REJECTED, () -> node.getReservedContainer() != null ? ActivityDiagnosticConstant. QUEUE_SKIPPED_BECAUSE_SINGLE_NODE_RESERVED : @@ -832,9 +832,9 @@ private CSAssignment assignContainersToChildQueues(Resource cluster, limits.addBlockedHeadroom(resourceToSubtract); if(LOG.isDebugEnabled()) { LOG.debug("Decrease parentLimits " + limits.getLimit() + - " for " + this.getQueueName() + " by " + + " for " + this.getQueuePath() + " by " + resourceToSubtract + " as childQueue=" + - childQueue.getQueueName() + " is blocked"); + childQueue.getQueuePath() + " is blocked"); } } } @@ -1129,7 +1129,7 @@ private void deriveCapacityFromAbsoluteConfigurations(String label, * (leafQueue.getUsersManager().getUserLimit() / 100.0f) * leafQueue.getUsersManager().getUserLimitFactor())); leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); - LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications=" + LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" + maxApplications + ", maxApplicationsPerUser=" + maxApplicationsPerUser + ", Abs Cap:" + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); @@ -1201,7 +1201,7 @@ public void attachContainer(Resource clusterResource, scheduler.getNode(rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, rmContainer.getContainer() .getResource(), node.getPartition()); - LOG.info("movedContainer" + " queueMoveIn=" + getQueueName() + LOG.info("movedContainer" + " queueMoveIn=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); @@ -1221,7 +1221,7 @@ public void detachContainer(Resource clusterResource, super.releaseResource(clusterResource, rmContainer.getContainer().getResource(), node.getPartition()); - LOG.info("movedContainer" + " queueMoveOut=" + getQueueName() + LOG.info("movedContainer" + " queueMoveOut=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); @@ -1316,8 +1316,8 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL, null, false); LOG.info("Killed container=" + toKillContainer.getContainerId() - + " from queue=" + lq.getQueueName() + " to make queue=" + this - .getQueueName() + "'s max-capacity enforced"); + + " from queue=" + lq.getQueuePath() + " to make queue=" + this + .getQueuePath() + "'s max-capacity enforced"); } if (!killableContainerIter.hasNext()) { @@ -1343,7 +1343,7 @@ public void apply(Resource cluster, allocateResource(cluster, allocation.getAllocatedOrReservedResource(), schedulerContainer.getNodePartition()); - LOG.info("assignedContainer" + " queue=" + getQueueName() + LOG.info("assignedContainer" + " queue=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + cluster); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java index 64ba578ff73..9e223e4153d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java @@ -124,7 +124,7 @@ public int hashCode() { @Override public String toString() { - return "QueueManagementChange{" + "queue=" + queue.getQueueName() + return "QueueManagementChange{" + "queue=" + queue.getQueuePath() + ", updatedEntitlementsByPartition=" + queueTemplateUpdate + ", queueAction=" + queueAction + ", transitionToQueueState=" + transitionToQueueState + '}'; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java index 96025588487..e6527ae4aa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java @@ -208,7 +208,7 @@ public void editSchedule() { policyClazz.getClass().getName(), clock.getTime() - startTime); if (queueManagementChanges.size() > 0) { LOG.debug(" Updated queue management changes for parent queue" + " " - + "{}: [{}]", parentQueue.getQueueName(), + + "{}: [{}]", parentQueue.getQueuePath(), queueManagementChanges.size() < 25 ? queueManagementChanges.toString() : queueManagementChanges.size()); @@ -218,7 +218,7 @@ public void editSchedule() { LOG.error( "Could not compute child queue management updates for parent " + "queue " - + parentQueue.getQueueName(), e); + + parentQueue.getQueuePath(), e); } } else{ LOG.debug("Skipping queue management updates for parent queue {} " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index b0700e8bf41..00b3a77d6fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -789,7 +789,7 @@ private Resource computeUserLimit(String userName, Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("User limit computation for " + userName - + ", in queue: " + lQueue.getQueueName() + + ", in queue: " + lQueue.getQueuePath() + ", userLimitPercent=" + lQueue.getUserLimit() + ", userLimitFactor=" + lQueue.getUserLimitFactor() + ", required=" + required diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index d91f4887db4..a2dbbb1ed8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -311,7 +311,7 @@ public void init(final CapacitySchedulerContext schedulerContext, LOG.info( "Initialized queue management policy for parent queue " + parentQueue - .getQueueName() + " with leaf queue template capacities : [" + .getQueuePath() + " with leaf queue template capacities : [" + leafQueueTemplate.getQueueCapacities() + "]"); } @@ -328,10 +328,10 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) if (!parentQueueLabels.contains(nodeLabel)) { LOG.error("Invalid node label " + nodeLabel + " on configured leaf template on parent" + " queue " + parentQueue - .getQueueName()); + .getQueuePath()); throw new IOException("Invalid node label " + nodeLabel + " on configured leaf template on parent" + " queue " + parentQueue - .getQueueName()); + .getQueuePath()); } } @@ -386,7 +386,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) if ( deactivatedLeafQueues.size() > 0) { LOG.debug("Parent queue = {}, " + ", nodeLabel = {}, deactivated leaf queues = [{}] ", - managedParentQueue.getQueueName(), nodeLabel, + managedParentQueue.getQueuePath(), nodeLabel, deactivatedLeafQueues.size() > 25 ? deactivatedLeafQueues .size() : deactivatedLeafQueues); @@ -405,7 +405,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) + deactivatedCapacity + EPSILON; if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueueName() + LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() + ", nodeLabel = " + nodeLabel + ", absCapacity = " + parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = " + leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = " @@ -422,7 +422,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) pendingApps.size()); if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueueName() + LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() + " : Found " + maxLeafQueuesTobeActivated + " leaf queues" + " to be activated with " + pendingApps.size() + " apps "); } @@ -507,7 +507,7 @@ void updateLeafQueueState() { String partition = e.getKey(); if (!newPartitions.contains(partition)) { itr.remove(); - LOG.info(managedParentQueue.getQueueName() + + LOG.info(managedParentQueue.getQueuePath() + " : Removed partition " + partition + " from leaf queue " + "state"); } else{ @@ -518,7 +518,7 @@ void updateLeafQueueState() { String queue = queueItr.next().getKey(); if (!newQueues.contains(queue)) { queueItr.remove(); - LOG.info(managedParentQueue.getQueueName() + " : Removed queue" + LOG.info(managedParentQueue.getQueuePath() + " : Removed queue" + queue + " from " + "leaf queue " + "state from partition " + partition); @@ -670,14 +670,14 @@ public void commitQueueManagementChanges( getCapacity(nodeLabel) > 0) { if (isActive(leafQueue, nodeLabel)) { LOG.debug("Queue is already active. Skipping activation : {}", - leafQueue.getQueueName()); + leafQueue.getQueuePath()); } else{ activate(leafQueue, nodeLabel); } } else{ if (!isActive(leafQueue, nodeLabel)) { LOG.debug("Queue is already de-activated. Skipping " - + "de-activation : {}", leafQueue.getQueueName()); + + "de-activation : {}", leafQueue.getQueuePath()); } else{ deactivate(leafQueue, nodeLabel); } @@ -744,7 +744,7 @@ public void reinitialize(CapacitySchedulerContext schedulerContext, LOG.info( "Reinitialized queue management policy for parent queue " + parentQueue - .getQueueName() + " with leaf queue template " + "capacities : [" + .getQueuePath() + " with leaf queue template " + "capacities : [" + leafQueueTemplate.getQueueCapacities() + "]"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 4ee73f2190e..4f35e0ac89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -128,7 +128,7 @@ public void setUp() throws IOException { when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); root = CapacitySchedulerQueueManager .parseQueue(csContext, csConf, null, "root", queues, queues, @@ -295,8 +295,8 @@ public void testLimitsComputation() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -368,7 +368,7 @@ public void testLimitsComputation() throws Exception { csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -392,7 +392,7 @@ public void testLimitsComputation() throws Exception { csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -603,8 +603,8 @@ public void testHeadroom() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index a5b9c38c6ff..9ff066d9ef5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -789,7 +789,7 @@ public void testHeadroom() throws Exception { Resource clusterResource = Resources.createResource(160 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java new file mode 100644 index 00000000000..04c03ff8b63 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java @@ -0,0 +1,13 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.junit.Test; + +import java.util.HashMap; + +import static org.junit.Assert.*; + +public class TestCSQueueStore { + @Test + public void testMapping() { + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index d568220dc68..aa7aec13569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -222,8 +222,8 @@ private void setupSortedQueues(CapacitySchedulerConfiguration conf) { public void testSortedQueues() throws Exception { // Setup queue configs setupSortedQueues(csConf); - Map queues = new HashMap(); - CSQueue root = + CSQueueStore queues = new CSQueueStore(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 03c1d604ab2..08fc97745fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -129,7 +129,7 @@ CapacitySchedulerContext csContext; CSQueue root; - Map queues; + CSQueueStore queues; final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -158,7 +158,7 @@ private void setUpWithNodeLabels() throws Exception { private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) throws Exception { CapacityScheduler spyCs = new CapacityScheduler(); - queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); cs = spy(spyCs); rmContext = TestUtils.getMockRMContext(); spyRMContext = spy(rmContext); @@ -2310,7 +2310,7 @@ public void testRackLocalityDelayScheduling() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); @@ -2746,7 +2746,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -2773,7 +2773,7 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -3120,7 +3120,7 @@ public void testLocalityConstraints() throws Exception { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); final String newRootName = "root" + System.currentTimeMillis(); setupQueueConfiguration(csConf, newRootName, false); @@ -3149,7 +3149,7 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 4ef9f7a5f7e..e0ddf457b8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -251,8 +251,8 @@ private void verifyQueueMetrics(CSQueue queue, public void testSingleLevelQueues() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -375,7 +375,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + "b"; csConf.setCapacity(Q_B, 70.5F); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); boolean exceptionOccurred = false; try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, @@ -496,8 +496,8 @@ public void testMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -662,8 +662,8 @@ public void testQueueCapacitySettingChildZero() throws Exception { csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -680,7 +680,7 @@ public void testQueueCapacitySettingParentZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -701,7 +701,7 @@ public void testQueueCapacityZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -717,7 +717,7 @@ public void testOffSwitchScheduling() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -799,7 +799,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); //B3 - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -901,7 +901,7 @@ public void testQueueAcl() throws Exception { final String Q_C11= Q_C + "." + C1 + "." + C11; csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -960,7 +960,7 @@ public void testAbsoluteResourceWithChangeInClusterResource() // Setup queue configs setupSingleLevelQueuesWithAbsoluteResource(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index da0e9c9e6fc..93132a95115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -95,8 +95,8 @@ private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); CSQueue root; - Map queues = new HashMap(); - Map oldQueues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); + CSQueueStore oldQueues = new CSQueueStore(); final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -546,7 +546,7 @@ public void testReservationNoContinueLook() throws Exception { // Test that with reservations-continue-look-all-nodes feature off // we don't unreserve and show we could get stuck - queues = new HashMap(); + queues = new CSQueueStore(); // test that the deadlock occurs when turned off CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, @@ -1188,7 +1188,7 @@ public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, csConf.setBoolean( CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); -- 2.20.1 (Apple Git-117)