From 2632cd4095ef2be2609d7706316663578f5c421a Mon Sep 17 00:00:00 2001 From: Johan Gustavsson Date: Thu, 20 Aug 2015 14:16:05 +0900 Subject: [PATCH] deal with performance issues related to a lot of queues --- .../scheduler/fair/QueueManager.java | 38 +++++++++++++++------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 6556717..e0d8a13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -86,7 +86,11 @@ public void initialize(Configuration conf) throws IOException, * could be referred to as just "parent1.queue2". */ public FSLeafQueue getLeafQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.LEAF); + return getLeafQueue(name, create, true); + } + + public FSLeafQueue getLeafQueue(String name, boolean create, boolean recalculate) { + FSQueue queue = getQueue(name, create, FSQueueType.LEAF, recalculate); if (queue instanceof FSParentQueue) { return null; } @@ -116,14 +120,18 @@ public boolean removeLeafQueue(String name) { * could be referred to as just "parent1.queue2". */ public FSParentQueue getParentQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.PARENT); + return getParentQueue(name, create, true); + } + + public FSParentQueue getParentQueue(String name, boolean create, boolean recalculate) { + FSQueue queue = getQueue(name, create, FSQueueType.PARENT, recalculate); if (queue instanceof FSLeafQueue) { return null; } return (FSParentQueue) queue; } - - private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { + + private FSQueue getQueue(String name, boolean create, FSQueueType queueType, boolean recalculate) { name = ensureRootPrefix(name); synchronized (queues) { FSQueue queue = queues.get(name); @@ -132,7 +140,7 @@ private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { queue = createQueue(name, queueType); // Update steady fair share for all queues - if (queue != null) { + if (queue != null && recalculate) { rootQueue.recomputeSteadyShares(); } } @@ -372,19 +380,25 @@ private String ensureRootPrefix(String name) { public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist - for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { - getLeafQueue(name, true); + synchronized (queues) { + for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { + getLeafQueue(name, true, false); + } } + rootQueue.recomputeSteadyShares(); } // At this point all leaves and 'parents with at least one child' would have been created. // Now create parents with no configured leaf. - for (String name : queueConf.getConfiguredQueues().get( - FSQueueType.PARENT)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { - getParentQueue(name, true); + synchronized (queues) { + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.PARENT)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { + getParentQueue(name, true, false); + } } + rootQueue.recomputeSteadyShares(); } for (FSQueue queue : queues.values()) { -- 2.3.2 (Apple Git-55)