diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 246ade78846..52845e41ff6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; @@ -180,18 +181,20 @@ private ApplicationPlacementContext getPlacementForUser(String user) if (mapping.getParentQueue() != null && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { - return getPlacementContext( + QueueMapping queueMapping = new QueueMapping(mapping.getType(), mapping.getSource(), - CURRENT_USER_MAPPING, groups.getGroups(user).get(0)), - user); + user, groups.getGroups(user).get(0)); + validateQueueMapping(queueMapping); + return getPlacementContext(queueMapping, user); } else if (mapping.getParentQueue() != null && mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING) && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { String secondaryGroup = getSecondaryGroup(user); if (secondaryGroup != null) { - return getPlacementContext(new QueueMapping(mapping.getType(), - mapping.getSource(), CURRENT_USER_MAPPING, secondaryGroup), - user); + QueueMapping queueMapping = new QueueMapping(mapping.getType(), + mapping.getSource(), user, secondaryGroup); + validateQueueMapping(queueMapping); + return getPlacementContext(queueMapping, user); } else { if (LOG.isDebugEnabled()) { LOG.debug("User {} is not associated with any Secondary Group. " @@ -497,6 +500,28 @@ private static void validateParentQueue(CSQueue parentQueue, } } + private void validateQueueMapping(QueueMapping queueMapping) + throws IOException { + String parentQueueName = queueMapping.getParentQueue(); + String leafQueueName = queueMapping.getQueue(); + CSQueue parentQueue = queueManager.getQueue(parentQueueName); + CSQueue leafQueue = queueManager.getQueue(leafQueueName); + + if (leafQueue == null || (!(leafQueue instanceof LeafQueue))) { + throw new IOException("mapping contains invalid or non-leaf queue : " + + leafQueueName); + } else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) { + throw new IOException( + "mapping contains invalid parent queue [" + parentQueueName + "]"); + } else if (!parentQueue.getQueueName() + .equals(leafQueue.getParent().getQueueName())) { + throw new IOException("mapping contains invalid parent queue " + + "which does not match existing leaf queue's parent : [" + + parentQueue.getQueueName() + "] does not match [ " + + leafQueue.getParent().getQueueName() + "]"); + } + } + @VisibleForTesting public List getQueueMappings() { return mappings; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 8e68984204a..6fcaf107974 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -113,15 +113,24 @@ public static final String C = CapacitySchedulerConfiguration.ROOT + ".c"; public static final String D = CapacitySchedulerConfiguration.ROOT + ".d"; public static final String E = CapacitySchedulerConfiguration.ROOT + ".e"; + public static final String ASUBGROUP1 = + CapacitySchedulerConfiguration.ROOT + ".asubgroup1"; + public static final String AGROUP = + CapacitySchedulerConfiguration.ROOT + ".egroup"; public static final String A1 = A + ".a1"; public static final String A2 = A + ".a2"; public static final String B1 = B + ".b1"; public static final String B2 = B + ".b2"; public static final String B3 = B + ".b3"; + public static final String ASUBGROUP1_A = ASUBGROUP1 + ".a"; + public static final String AGROUP_A = AGROUP + ".e"; public static final float A_CAPACITY = 20f; - public static final float B_CAPACITY = 40f; + public static final float B_CAPACITY = 20f; public static final float C_CAPACITY = 20f; public static final float D_CAPACITY = 20f; + public static final float ASUBGROUP1_CAPACITY = 10f; + public static final float AGROUP_CAPACITY = 10f; + public static final float A1_CAPACITY = 30; public static final float A2_CAPACITY = 70; public static final float B1_CAPACITY = 60f; @@ -329,12 +338,15 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( // Define top-level queues // Set childQueue for root conf.setQueues(ROOT, - new String[] { "a", "b", "c", "d", "asubgroup1", "asubgroup2" }); + new String[] { "a", "b", "c", "d", "asubgroup1", "asubgroup2", + "egroup" }); conf.setCapacity(A, A_CAPACITY); conf.setCapacity(B, B_CAPACITY); conf.setCapacity(C, C_CAPACITY); conf.setCapacity(D, D_CAPACITY); + conf.setCapacity(ASUBGROUP1, ASUBGROUP1_CAPACITY); + conf.setCapacity(AGROUP, AGROUP_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1", "a2" }); @@ -351,6 +363,13 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); + conf.setQueues(ASUBGROUP1, new String[] { "a" }); + conf.setCapacity(ASUBGROUP1_A, 100f); + conf.setUserLimitFactor(ASUBGROUP1_A, 100.0f); + conf.setQueues(AGROUP, new String[] { "e" }); + conf.setCapacity(AGROUP_A, 100f); + conf.setUserLimitFactor(AGROUP_A, 100.0f); + conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index c18c2465c70..13390e7662b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -226,7 +226,7 @@ public void testNestedUserQueueWithPrimaryGroupAsDynamicParentQueue() queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); - testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true); + testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "e"); } @Test @@ -259,12 +259,12 @@ public void testNestedUserQueueWithSecondaryGroupAsDynamicParentQueue() queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping1); - testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, false); + testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, false, "a"); } private void testNestedUserQueueWithDynamicParentQueue( - List mapping, - boolean primary) + List mapping, boolean primary, + String user) throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); @@ -301,13 +301,13 @@ private void testNestedUserQueueWithDynamicParentQueue( UserGroupMappingPlacementRule r = (UserGroupMappingPlacementRule) rules.get(0); - ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "a"); - assertEquals("Queue", "a", ctx.getQueue()); + ApplicationPlacementContext ctx = r.getPlacementForApp(asc, user); + assertEquals("Queue", user, ctx.getQueue()); if (primary) { - assertEquals("Primary Group", "agroup", ctx.getParentQueue()); + assertEquals("Primary Group", user + "group", ctx.getParentQueue()); } else { - assertEquals("Secondary Group", "asubgroup1", ctx.getParentQueue()); + assertEquals("Secondary Group", user + "subgroup1", ctx.getParentQueue()); } mockRM.close(); }