From a5fba8835b93416156eb0a26875ff4312ed83f09 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Tue, 21 Jan 2020 17:22:03 +0100 Subject: [PATCH] YARN-9879 Allow multiple leaf queues with the same name in CS --- .../scheduler/capacity/AbstractCSQueue.java | 9 +- .../scheduler/capacity/CSQueue.java | 8 +- .../scheduler/capacity/CSQueueStore.java | 115 ++++++++++++++++++ .../scheduler/capacity/CapacityScheduler.java | 2 +- .../CapacitySchedulerQueueManager.java | 81 ++++++------ .../capacity/TestApplicationLimits.java | 14 +-- .../TestApplicationLimitsByPartition.java | 2 +- .../scheduler/capacity/TestCSQueueStore.java | 13 ++ .../capacity/TestChildQueueOrder.java | 4 +- .../scheduler/capacity/TestLeafQueue.java | 14 +-- .../scheduler/capacity/TestParentQueue.java | 26 ++-- .../scheduler/capacity/TestReservations.java | 8 +- 12 files changed, 214 insertions(+), 82 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 3ae22ad0ebd..a6147fa2c13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -193,7 +193,7 @@ protected void setupConfigurableCapacities( public String getQueuePath() { return queuePath; } - + @Override public float getCapacity() { return queueCapacities.getCapacity(); @@ -244,10 +244,15 @@ public CSQueueMetrics getMetrics() { } @Override - public String getQueueName() { + public String getQueueShortName() { return queueName; } + @Override + public String getQueueName() { + return queuePath; + } + @Override public PrivilegedEntity getPrivilegedEntity() { return queueEntity; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index d507e53543c..da6c5ea1fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -73,11 +73,17 @@ public void setParent(CSQueue newParentQueue); /** - * Get the queue name. + * Get the queue's internal reference name. * @return the queue name */ public String getQueueName(); + /** + * Get the queue's legacy name + * @return the queue name + */ + public String getQueueShortName(); + /** * Get the full name of the queue, including the heirarchy. * @return the full name of the queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java new file mode 100644 index 00000000000..0817c44f699 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueStore.java @@ -0,0 +1,115 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import com.google.common.collect.ImmutableMap; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class CSQueueStore { + private final Map fullNameQueues = new ConcurrentHashMap<>(); + private final Map shortNameToFullName = new HashMap<>(); + private final Map> ambiguousShortNames = new HashMap<>(); + + public Map getFullNameQueues() { + return ImmutableMap.copyOf(fullNameQueues); + } + + //we must synchronize here because we need to maintain multiple maps to be + //in sync, and concurrent hashMap does not help with that + private synchronized void addShortNameMapping(String shortName, String fullName) { + if (ambiguousShortNames.containsKey(shortName)) { + //if the short name is already ambiguous we just add it to the list + ambiguousShortNames.get(shortName).add(fullName); + } else if (shortNameToFullName.containsKey(shortName) + && !fullName.equals(shortNameToFullName.get(shortName))) { + + //the short name became ambiguous creating the ambiguous list + Set ambiguousQueues = new HashSet<>(); + ambiguousQueues.add(fullName); + ambiguousQueues.add(shortNameToFullName.get(shortName)); + ambiguousShortNames.put(shortName, ambiguousQueues); + + //removing the short name from queue name map, since there is no + //disambiguous mapping + shortNameToFullName.remove(shortName); + } else { + //there is no issue with the queue's short name we just add it to the map + shortNameToFullName.put(shortName, fullName); + } + } + + public void add(CSQueue queue) { + String fullName = queue.getQueueName(); + String shortName = queue.getQueueShortName(); + + fullNameQueues.put(fullName, queue); + if (queue instanceof LeafQueue) { + addShortNameMapping(shortName, fullName); + } + } + + public synchronized void remove(CSQueue queue) { + //if no queue is specified, we can consider it already removed, also consistent + //with hashmap behaviour, so no new issues will be caused by it + if (queue == null) { + return; + } + + String fullName = queue.getQueueName(); + String shortName = queue.getQueueShortName(); + + //removing from the full and short name maps as well + fullNameQueues.remove(fullName); + if (shortNameToFullName.containsKey(shortName)) { + shortNameToFullName.remove(shortName); + } + + if (ambiguousShortNames.containsKey(shortName) && queue instanceof LeafQueue) { + //if the removed queued was an ambiguous one, we check if removing it + //resolves the issue. + Set conflictingQueues = ambiguousShortNames.get(shortName); + conflictingQueues.remove(fullName); + if (conflictingQueues.size() == 1) { + //if we have only 1 conflictingQueue, it is not a conflict, resolving + ambiguousShortNames.remove(shortName); + shortNameToFullName.put(shortName, conflictingQueues.iterator().next()); + } + } + + } + + public synchronized void remove(String name) { + CSQueue queue = get(name); + if (queue != null) remove(queue); + } + + public int getQueueCount() { + return fullNameQueues.size(); + } + + public CSQueue get(String name) { + CSQueue ret = fullNameQueues.getOrDefault(name, null); + + if (ret == null) { + String mappedName = shortNameToFullName.getOrDefault(name, null); + if (mappedName != null) { + ret = fullNameQueues.get(mappedName); + } + } + return ret; + } + + public synchronized void clear() { + fullNameQueues.clear(); + shortNameToFullName.clear(); + ambiguousShortNames.clear(); + } + + public Collection getQueues() { + return fullNameQueues.values(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index fcc05602871..af12d57b3c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2396,7 +2396,7 @@ public void addQueue(Queue queue) AbstractManagedParentQueue parent = (AbstractManagedParentQueue) newQueue.getParent(); - String queuename = newQueue.getQueueName(); + String queuename = newQueue.getQueuePath(); parent.addChildQueue(newQueue); this.queueManager.addQueue(queuename, newQueue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index d9b3ebd2eda..d698b71acf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +90,7 @@ public CSQueue hook(CSQueue queue) { private static final QueueHook NOOP = new QueueHook(); private CapacitySchedulerContext csContext; private final YarnAuthorizationProvider authorizer; - private final Map queues = new ConcurrentHashMap<>(); + private final CSQueueStore queues = new CSQueueStore(); private CSQueue root; private final RMNodeLabelsManager labelManager; private AppPriorityACLsManager appPriorityACLManager; @@ -121,7 +120,7 @@ public CSQueue getRootQueue() { @Override public Map getQueues() { - return queues; + return queues.getFullNameQueues(); } @Override @@ -131,7 +130,7 @@ public void removeQueue(String queueName) { @Override public void addQueue(String queueName, CSQueue queue) { - this.queues.put(queueName, queue); + this.queues.add(queue); } @Override @@ -167,7 +166,7 @@ public void initializeQueues(CapacitySchedulerConfiguration conf) public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues - Map newQueues = new HashMap<>(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); @@ -212,8 +211,9 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) static CSQueue parseQueue( CapacitySchedulerContext csContext, CapacitySchedulerConfiguration conf, - CSQueue parent, String queueName, Map queues, - Map oldQueues, + CSQueue parent, String queueName, + CSQueueStore queues, + CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; String fullQueueName = (parent == null) ? @@ -232,7 +232,7 @@ static CSQueue parseQueue( // system if (isReservableQueue) { queue = new PlanQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); //initializing the "internal" default queue, for SLS compatibility String defReservationId = @@ -248,31 +248,31 @@ static CSQueue parseQueue( } childQueues.add(resQueue); ((PlanQueue) queue).setChildQueues(childQueues); - queues.put(defReservationId, resQueue); + queues.add(resQueue); } else if (isAutoCreateEnabled) { queue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ queue = new LeafQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); // Used only for unit tests queue = hook.hook(queue); } } else{ if (isReservableQueue) { throw new IllegalStateException( - "Only Leaf Queues can be reservable for " + queueName); + "Only Leaf Queues can be reservable for " + fullQueueName); } ParentQueue parentQueue; if (isAutoCreateEnabled) { parentQueue = new ManagedParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } else{ parentQueue = new ParentQueue(csContext, queueName, parent, - oldQueues.get(queueName)); + oldQueues.get(fullQueueName)); } // Used only for unit tests @@ -288,14 +288,14 @@ static CSQueue parseQueue( } - if (queue instanceof LeafQueue && queues.containsKey(queueName) && queues - .get(queueName) instanceof LeafQueue) { - throw new IOException("Two leaf queues were named " + queueName + if (queue instanceof LeafQueue && queues.get(fullQueueName) != null && queues + .get(fullQueueName) instanceof LeafQueue) { + throw new IOException("Two leaf queues were named " + fullQueueName + ". Leaf queue names must be distinct"); } - queues.put(queueName, queue); + queues.add(queue); - LOG.info("Initialized queue: " + queue); + LOG.info("Initialized queue: " + fullQueueName); return queue; } @@ -308,15 +308,14 @@ static CSQueue parseQueue( * @param queues existing queues * @param newQueues new queues */ - private void validateQueueHierarchy(Map queues, - Map newQueues, CapacitySchedulerConfiguration newConf) + private void validateQueueHierarchy(CSQueueStore queues, + CSQueueStore newQueues, CapacitySchedulerConfiguration newConf) throws IOException { // check that all static queues are included in the newQueues list - for (Map.Entry e : queues.entrySet()) { - if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue() + for (CSQueue oldQueue : queues.getQueues()) { + if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(oldQueue .getClass()))) { - String queueName = e.getKey(); - CSQueue oldQueue = e.getValue(); + String queueName = oldQueue.getQueueName(); CSQueue newQueue = newQueues.get(queueName); if (null == newQueue) { // old queue doesn't exist in the new XML @@ -391,27 +390,21 @@ private void validateQueueHierarchy(Map queues, * @param existingQueues, the existing queues * @param newQueues the new queues based on new XML */ - private void updateQueues(Map existingQueues, - Map newQueues) { + private void updateQueues(CSQueueStore existingQueues, + CSQueueStore newQueues) { CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - for (Map.Entry e : newQueues.entrySet()) { - String queueName = e.getKey(); - CSQueue queue = e.getValue(); - if (!existingQueues.containsKey(queueName)) { - existingQueues.put(queueName, queue); + for (CSQueue queue : newQueues.getQueues()) { + if (existingQueues.get(queue.getQueueName()) == null) { + existingQueues.add(queue); } } - for (Iterator> itr = existingQueues.entrySet() - .iterator(); itr.hasNext();) { - Map.Entry e = itr.next(); - String queueName = e.getKey(); - CSQueue existingQueue = e.getValue(); - - if (!newQueues.containsKey(queueName) && !( - existingQueue instanceof AutoCreatedLeafQueue && conf + + for (CSQueue queue: existingQueues.getQueues()) { + if (existingQueues.get(queue.getQueueName()) == null && !( + queue instanceof AutoCreatedLeafQueue && conf .isAutoCreateChildQueueEnabled( - existingQueue.getParent().getQueuePath()))) { - itr.remove(); + queue.getParent().getQueuePath()))) { + existingQueues.remove(queue); } } } @@ -424,10 +417,10 @@ private void updateQueues(Map existingQueues, * @throws IOException if fails to set queue acls */ public static void setQueueAcls(YarnAuthorizationProvider authorizer, - AppPriorityACLsManager appPriorityACLManager, Map queues) + AppPriorityACLsManager appPriorityACLManager, CSQueueStore queues) throws IOException { List permissions = new ArrayList<>(); - for (CSQueue queue : queues.values()) { + for (CSQueue queue : queues.getQueues()) { AbstractCSQueue csQueue = (AbstractCSQueue) queue; permissions.add( new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 4ee73f2190e..4f35e0ac89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -128,7 +128,7 @@ public void setUp() throws IOException { when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); root = CapacitySchedulerQueueManager .parseQueue(csContext, csConf, null, "root", queues, queues, @@ -295,8 +295,8 @@ public void testLimitsComputation() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -368,7 +368,7 @@ public void testLimitsComputation() throws Exception { csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -392,7 +392,7 @@ public void testLimitsComputation() throws Exception { csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999); // Re-create queues to get new configs. - queues = new HashMap(); + queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); @@ -603,8 +603,8 @@ public void testHeadroom() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index a5b9c38c6ff..9ff066d9ef5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -789,7 +789,7 @@ public void testHeadroom() throws Exception { Resource clusterResource = Resources.createResource(160 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java new file mode 100644 index 00000000000..04c03ff8b63 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java @@ -0,0 +1,13 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.junit.Test; + +import java.util.HashMap; + +import static org.junit.Assert.*; + +public class TestCSQueueStore { + @Test + public void testMapping() { + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index d568220dc68..aa7aec13569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -222,8 +222,8 @@ private void setupSortedQueues(CapacitySchedulerConfiguration conf) { public void testSortedQueues() throws Exception { // Setup queue configs setupSortedQueues(csConf); - Map queues = new HashMap(); - CSQueue root = + CSQueueStore queues = new CSQueueStore(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 03c1d604ab2..08fc97745fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -129,7 +129,7 @@ CapacitySchedulerContext csContext; CSQueue root; - Map queues; + CSQueueStore queues; final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -158,7 +158,7 @@ private void setUpWithNodeLabels() throws Exception { private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) throws Exception { CapacityScheduler spyCs = new CapacityScheduler(); - queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); cs = spy(spyCs); rmContext = TestUtils.getMockRMContext(); spyRMContext = spy(rmContext); @@ -2310,7 +2310,7 @@ public void testRackLocalityDelayScheduling() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); @@ -2746,7 +2746,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -2773,7 +2773,7 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, @@ -3120,7 +3120,7 @@ public void testLocalityConstraints() throws Exception { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); final String newRootName = "root" + System.currentTimeMillis(); setupQueueConfiguration(csConf, newRootName, false); @@ -3149,7 +3149,7 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 4ef9f7a5f7e..e0ddf457b8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -251,8 +251,8 @@ private void verifyQueueMetrics(CSQueue queue, public void testSingleLevelQueues() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -375,7 +375,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + "b"; csConf.setCapacity(Q_B, 70.5F); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); boolean exceptionOccurred = false; try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, @@ -496,8 +496,8 @@ public void testMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -662,8 +662,8 @@ public void testQueueCapacitySettingChildZero() throws Exception { csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); - - Map queues = new HashMap(); + + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -680,7 +680,7 @@ public void testQueueCapacitySettingParentZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -701,7 +701,7 @@ public void testQueueCapacityZero() throws Exception { final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -717,7 +717,7 @@ public void testOffSwitchScheduling() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -799,7 +799,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); //B3 - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -901,7 +901,7 @@ public void testQueueAcl() throws Exception { final String Q_C11= Q_C + "." + C1 + "." + C11; csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, @@ -960,7 +960,7 @@ public void testAbsoluteResourceWithChangeInClusterResource() // Setup queue configs setupSingleLevelQueuesWithAbsoluteResource(csConf); - Map queues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index da0e9c9e6fc..93132a95115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -95,8 +95,8 @@ private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); CSQueue root; - Map queues = new HashMap(); - Map oldQueues = new HashMap(); + CSQueueStore queues = new CSQueueStore(); + CSQueueStore oldQueues = new CSQueueStore(); final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -546,7 +546,7 @@ public void testReservationNoContinueLook() throws Exception { // Test that with reservations-continue-look-all-nodes feature off // we don't unreserve and show we could get stuck - queues = new HashMap(); + queues = new CSQueueStore(); // test that the deadlock occurs when turned off CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, @@ -1188,7 +1188,7 @@ public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, csConf.setBoolean( CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); - Map newQueues = new HashMap(); + CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); -- 2.20.1 (Apple Git-117)