From 2fccb5e4c1e2626e2ac0d9e1a5d4c0aabcc1144e Mon Sep 17 00:00:00 2001 From: keyki Date: Thu, 3 Jul 2014 23:11:50 +0200 Subject: [PATCH] YARN-2250. Fix finding lowest common ancestor of target and source queue during app move --- .../scheduler/fair/FairScheduler.java | 14 ++--- .../scheduler/fair/TestFairScheduler.java | 66 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 0cbcaae..5af33c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import com.google.common.base.Preconditions; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * A scheduler that schedules resources between a set of queues. The scheduler @@ -1468,8 +1468,9 @@ private void executeMove(SchedulerApplication app, maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, oldQueue); } } - - private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { + + @VisibleForTesting + FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { // Because queue names include ancestors, separated by periods, we can find // the lowest common ancestors by going from the start of the names until // there's a character that doesn't match. @@ -1479,13 +1480,12 @@ private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { // when the queues are root.applepie and root.appletart int lastPeriodIndex = -1; for (int i = 0; i < Math.max(name1.length(), name2.length()); i++) { - if (name1.length() <= i || name2.length() <= i || - name1.charAt(i) != name2.charAt(i)) { - return queueMgr.getQueue(name1.substring(lastPeriodIndex)); + if (name1.length() <= i || name2.length() <= i || name1.charAt(i) != name2.charAt(i)) { + return queueMgr.getQueue(name1.substring(0, lastPeriodIndex)); } else if (name1.charAt(i) == '.') { lastPeriodIndex = i; } } - return queue1; // names are identical + return queue1; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index a54387a..0ab61cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -25,6 +25,8 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.io.FileWriter; @@ -3077,4 +3079,68 @@ public void testMoveToNonexistentQueue() throws Exception { createSchedulingRequest(1024, 1, "queue1", "user1", 3); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); } + + @Test + public void testLowestCommonAncestorForNonRootParent() throws IOException, YarnException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSLeafQueue aQueue = mock(FSLeafQueue.class); + FSLeafQueue bQueue = mock(FSLeafQueue.class); + when(aQueue.getName()).thenReturn("root.queue1.a"); + when(bQueue.getName()).thenReturn("root.queue1.b"); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); + assertEquals(ancestorQueue, queue1); + } + + @Test + public void testLowestCommonAncestorRootParent() throws IOException, YarnException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSLeafQueue aQueue = mock(FSLeafQueue.class); + FSLeafQueue bQueue = mock(FSLeafQueue.class); + when(aQueue.getName()).thenReturn("root.a"); + when(bQueue.getName()).thenReturn("root.b"); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("root", false); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); + assertEquals(ancestorQueue, queue1); + } + + @Test + public void testLowestCommonAncestorDeeperHierarchy() throws IOException, YarnException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSQueue aQueue = mock(FSLeafQueue.class); + FSQueue bQueue = mock(FSLeafQueue.class); + FSQueue a1Queue = mock(FSLeafQueue.class); + FSQueue b1Queue = mock(FSLeafQueue.class); + when(a1Queue.getName()).thenReturn("root.queue1.a.a1"); + when(b1Queue.getName()).thenReturn("root.queue1.b.b1"); + when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue)); + when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue)); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue); + assertEquals(ancestorQueue, queue1); + } } -- 1.8.5.2 (Apple Git-48)