diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 5f20cbeebb5..a67f5528abf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -169,6 +169,11 @@ private String getSecondaryGroup(String user) throws IOException { break; } } + + if (secondaryGroup == null && LOG.isDebugEnabled()) { + LOG.debug("User {} is not associated with any Secondary " + + "Group. Hence it may use the 'default' queue", user); + } return secondaryGroup; } @@ -193,10 +198,6 @@ private ApplicationPlacementContext getPlacementForUser(String user) mapping.getSource(), CURRENT_USER_MAPPING, secondaryGroup), user); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("User {} is not associated with any Secondary Group. " - + "Hence it may use the 'default' queue", user); - } return null; } } else if (mapping.queue.equals(CURRENT_USER_MAPPING)) { @@ -208,10 +209,6 @@ private ApplicationPlacementContext getPlacementForUser(String user) if (secondaryGroup != null) { return getPlacementContext(mapping, secondaryGroup); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("User {} is not associated with any Secondary " - + "Group. Hence it may use the 'default' queue", user); - } return null; } } else { @@ -221,6 +218,13 @@ private ApplicationPlacementContext getPlacementForUser(String user) if (user.equals(mapping.source)) { if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); + } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { + String secondaryGroup = getSecondaryGroup(user); + if (secondaryGroup != null) { + return getPlacementContext(mapping, secondaryGroup); + } else { + return null; + } } else { return getPlacementContext(mapping); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 8e68984204a..d62a2bfe853 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -118,6 +118,7 @@ public static final String B1 = B + ".b1"; public static final String B2 = B + ".b2"; public static final String B3 = B + ".b3"; + public static final String B4 = B + ".b4subgroup1"; public static final float A_CAPACITY = 20f; public static final float B_CAPACITY = 40f; public static final float C_CAPACITY = 20f; @@ -126,7 +127,8 @@ public static final float A2_CAPACITY = 70; public static final float B1_CAPACITY = 60f; public static final float B2_CAPACITY = 20f; - public static final float B3_CAPACITY = 20f; + public static final float B3_CAPACITY = 10f; + public static final float B4_CAPACITY = 10f; public static final int NODE_MEMORY = 16; @@ -343,13 +345,15 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); - conf.setQueues(B, new String[] { "b1", "b2", "b3" }); + conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4subgroup1" }); conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); + conf.setCapacity(B4, B4_CAPACITY); + conf.setUserLimitFactor(B4, 100.0f); conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index 5d023960a4a..2ba3467dedf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -223,8 +223,15 @@ public void testNestedUserQueueWithPrimaryGroupAsDynamicParentQueue() UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", getQueueMapping("%secondary_group", "%user")); + // u:b4:%secondary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, + "b4", "%secondary_group"); + queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); + queueMappingsForUG.add(userQueueMapping3); testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true); } @@ -313,7 +320,7 @@ private void testNestedUserQueueWithDynamicParentQueue( } @Test - public void testDynamicPrimaryGroupQueue() throws Exception { + public void testFixedUserWithDynamicGroupQueue() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, @@ -344,8 +351,15 @@ public void testDynamicPrimaryGroupQueue() throws Exception { UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user2", "%primary_group"); + // u:b4:%secondary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4", + "%secondary_group"); + queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); + queueMappingsForUG.add(userQueueMapping3); existingMappingsForUG.addAll(queueMappingsForUG); conf.setQueueMappings(existingMappingsForUG); @@ -372,5 +386,8 @@ public void testDynamicPrimaryGroupQueue() throws Exception { ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2"); assertEquals("Queue", "user2group", ctx1.getQueue()); + + ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "b4"); + assertEquals("Queue", "b4subgroup1", ctx2.getQueue()); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index aa137c0c6eb..b37cf10a578 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -170,15 +170,19 @@ Example: ``` yarn.scheduler.capacity.queue-mappings - u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group,u:user3:%secondary_group,u:%user:%primary_group.%user,u:%user:%secondary_group.%user + u:%user:%primary_group.%user,u:%user:%secondary_group.%user,u:%user:%user,u:%user:%primary_group,u:%user:%secondary_group,u:user2:%primary_group,u:user3:%secondary_group,u:user1:queue1,g:group1:queue2 - Here, is mapped to , is mapped to , - maps users to queues with the same name as user, is mapped - to queue name same as , maps users to queue with the - same name as user but parent queue name should be same as - of the user, maps users to queue with the same name as user but parent - queue name should be same as any of the user - respectively. The mappings will be evaluated from left to + Here, maps users to queue with the same name as user but + parent queue name should be same as of + the user, maps users to queue with the same name as user + but parent queue name should be same as any + of the user, maps users to queues with the same name as user, + maps users to queue name same as of the user, + maps users to queue name same as of the user, + is mapped to queue name same as , + is mapped to queue name same as , + is mapped to , is mapped to + respectively.The mappings will be evaluated from left to right, and the first valid mapping will be used.