From bae7491bf234f53663782d0e477951374e716653 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Thu, 30 Apr 2020 17:01:25 +0200 Subject: [PATCH] YARN-10108. FS-CS converter: nestedUserQueue with default rule results in invalid queue mapping --- .../scheduler/capacity/CapacityScheduler.java | 6 ++- ...estCapacitySchedulerAutoQueueCreation.java | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cca4fe1872f..eb0bb9aa321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -989,12 +989,14 @@ private void addApplication(ApplicationId applicationId, String queueName, // not auto-created above, then its parent queue should match // the parent queue specified in queue mapping } else if (!queue.getParent().getQueueShortName().equals( - placementContext.getParentQueue())) { + placementContext.getParentQueue()) + && !queue.getParent().getQueuePath().equals( + placementContext.getParentQueue())) { String message = "Auto created Leaf queue " + placementContext.getQueue() + " " + "already exists under queue : " + queue .getParent().getQueueShortName() - + ".But Queue mapping configuration " + + + ". But Queue mapping configuration " + CapacitySchedulerConfiguration.QUEUE_MAPPING + " has been " + "updated to a different parent queue : " + placementContext.getParentQueue() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index a98abb9a004..087a91d1512 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -162,6 +162,60 @@ public void testAutoCreateLeafQueueCreation() throws Exception { } } + @Test(timeout = 20000) + public void testAutoCreateLeafQueueCreationUsingFullParentPath() + throws Exception { + + try { + // submit an app + submitApp(mockRM, cs.getQueue(PARENT_QUEUE), USER0, USER0, 1, 1); + + // check preconditions + List appsInC = cs.getAppsInQueue(PARENT_QUEUE); + assertEquals(1, appsInC.size()); + assertNotNull(cs.getQueue(USER0)); + + AutoCreatedLeafQueue autoCreatedLeafQueue = + (AutoCreatedLeafQueue) cs.getQueue(USER0); + ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue( + PARENT_QUEUE); + assertEquals(parentQueue, autoCreatedLeafQueue.getParent()); + + Map expectedChildQueueAbsCapacity = + populateExpectedAbsCapacityByLabelForParentQueue(1); + validateInitialQueueEntitlement(parentQueue, USER0, + expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC); + + validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000); + validateContainerLimits(autoCreatedLeafQueue); + + assertTrue(autoCreatedLeafQueue + .getOrderingPolicy() instanceof FairOrderingPolicy); + + setupGroupQueueMappings("root.d", cs.getConfiguration(), "%user"); + cs.reinitialize(cs.getConfiguration(), mockRM.getRMContext()); + + submitApp(mockRM, cs.getQueue("d"), TEST_GROUPUSER, TEST_GROUPUSER, 1, 1); + autoCreatedLeafQueue = + (AutoCreatedLeafQueue) cs.getQueue(TEST_GROUPUSER); + parentQueue = (ManagedParentQueue) cs.getQueue("d"); + assertEquals(parentQueue, autoCreatedLeafQueue.getParent()); + + expectedChildQueueAbsCapacity = + new HashMap() {{ + put(NO_LABEL, 0.02f); + }}; + + validateInitialQueueEntitlement(parentQueue, TEST_GROUPUSER, + expectedChildQueueAbsCapacity, + new HashSet() {{ add(NO_LABEL); }}); + + } finally { + cleanupQueue(USER0); + cleanupQueue(TEST_GROUPUSER); + } + } + @Test public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception { try { -- 2.26.2