diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 43a3043b4d0..569b1a3781e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -159,7 +159,14 @@ private ApplicationPlacementContext getPlacementForUser(String user) for (QueueMapping mapping : mappings) { if (mapping.type == MappingType.USER) { if (mapping.source.equals(CURRENT_USER_MAPPING)) { - if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + if (mapping.getParentQueue() != null + && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) + && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { + return getPlacementContext( + new QueueMapping(mapping.getType(), mapping.getSource(), + CURRENT_USER_MAPPING, groups.getGroups(user).get(0)), + user); + } else if (mapping.queue.equals(CURRENT_USER_MAPPING)) { return getPlacementContext(mapping, user); } else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); @@ -352,7 +359,12 @@ private static boolean ifQueueDoesNotExist(CSQueue queue) { private static QueueMapping validateAndGetAutoCreatedQueueMapping( CapacitySchedulerQueueManager queueManager, QueueMapping mapping, QueuePath queuePath) throws IOException { - if (queuePath.hasParentQueue()) { + if (queuePath.hasParentQueue() + && queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING)) { + // dynamic parent queue + return new QueueMapping(mapping.getType(), mapping.getSource(), + queuePath.getLeafQueue(), queuePath.getParentQueue()); + } else if (queuePath.hasParentQueue()) { //if parent queue is specified, // then it should exist and be an instance of ManagedParentQueue validateParentQueue(queueManager.getQueue(queuePath.getParentQueue()), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index a6143ba6cbb..324afd64ff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -51,6 +51,13 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, String inputQueue, String expectedQueue, boolean overwrite) throws YarnException { + verifyQueueMapping(queueMapping, inputUser, inputQueue, expectedQueue, + overwrite, null); + } + + private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, + String inputQueue, String expectedQueue, boolean overwrite, + String expectedParentQueue) throws YarnException { Groups groups = new Groups(conf); UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule( overwrite, Arrays.asList(queueMapping), groups); @@ -60,6 +67,10 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, ApplicationPlacementContext ctx = rule.getPlacementForApp(asc, inputUser); Assert.assertEquals(expectedQueue, ctx != null ? ctx.getQueue() : inputQueue); + if (expectedParentQueue != null) { + Assert.assertEquals(expectedParentQueue, + ctx != null ? ctx.getParentQueue() : inputQueue); + } } @Test @@ -74,6 +85,9 @@ public void testMapping() throws YarnException { "a", "a"); verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a", "agroup"); + verifyQueueMapping( + new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"), + "a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup"); verifyQueueMapping(new QueueMapping(MappingType.GROUP, "asubgroup1", "q1"), "a", "q1"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index b4f97123c0f..554210c0290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -18,12 +18,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; +import org.apache.hadoop.yarn.util.Records; import org.junit.Test; import java.util.ArrayList; @@ -33,6 +39,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.setupQueueConfiguration; import static org.hamcrest.CoreMatchers.hasItems; import static org.junit.Assert.assertThat; +import org.junit.Assert; public class TestCapacitySchedulerQueueMappingFactory { @@ -125,4 +132,125 @@ public void testUpdatePlacementRulesFactory() throws Exception { assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_USER_GROUP)); assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_APP_NAME)); } + + @Test + public void testNestedUserQueueWithDynamicParentQueue() throws Exception { + + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + + List queuePlacementRules = new ArrayList<>(); + queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); + conf.setQueuePlacementRules(queuePlacementRules); + + List existingMappingsForUG = + conf.getQueueMappings(); + + // set queue mapping + List queueMappingsForUG = + new ArrayList<>(); + + // u:%user:%primary_group.%user + UserGroupMappingPlacementRule.QueueMapping userQueueMapping = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", + getQueueMapping("%primary_group", "%user")); + queueMappingsForUG.add(userQueueMapping); + + existingMappingsForUG.addAll(queueMappingsForUG); + conf.setQueueMappings(existingMappingsForUG); + + // override with queue mappings + conf.setOverrideWithQueueMappings(true); + + mockRM = new MockRM(conf); + CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler(); + cs.updatePlacementRules(); + mockRM.start(); + cs.start(); + + ApplicationSubmissionContext asc = + Records.newRecord(ApplicationSubmissionContext.class); + asc.setQueue("default"); + String inputUser = "a"; + + List rules = + cs.getRMContext().getQueuePlacementManager().getPlacementRules(); + + UserGroupMappingPlacementRule r = + (UserGroupMappingPlacementRule) rules.get(0); + ApplicationPlacementContext ctx = r.getPlacementForApp(asc, inputUser); + Assert.assertEquals("a", ctx != null ? ctx.getQueue() : "default"); + Assert.assertEquals("agroup", + ctx != null ? ctx.getParentQueue() : "default"); + } + + @Test + public void testNestedUserQueueWithStaticParentQueue() throws Exception { + + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + + List queuePlacementRules = new ArrayList<>(); + queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); + conf.setQueuePlacementRules(queuePlacementRules); + + List existingMappingsForUG = + conf.getQueueMappings(); + + // set queue mapping + List queueMappingsForUG = + new ArrayList<>(); + + // u:user1:b1 + UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user1", + "b1"); + // u:%user:parentqueue.%user + UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", + getQueueMapping("c", "%user")); + queueMappingsForUG.add(userQueueMapping1); + queueMappingsForUG.add(userQueueMapping2); + + existingMappingsForUG.addAll(queueMappingsForUG); + conf.setQueueMappings(existingMappingsForUG); + + // override with queue mappings + conf.setOverrideWithQueueMappings(true); + + mockRM = new MockRM(conf); + CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler(); + cs.updatePlacementRules(); + mockRM.start(); + cs.start(); + + ApplicationSubmissionContext asc = + Records.newRecord(ApplicationSubmissionContext.class); + asc.setQueue("default"); + + List rules = + cs.getRMContext().getQueuePlacementManager().getPlacementRules(); + + UserGroupMappingPlacementRule r = + (UserGroupMappingPlacementRule) rules.get(0); + + ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1"); + Assert.assertEquals("b1", ctx != null ? ctx.getQueue() : "default"); + + ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user2"); + Assert.assertEquals("user2", ctx2 != null ? ctx2.getQueue() : "default"); + Assert.assertEquals("c", + ctx2 != null ? ctx2.getParentQueue() : "default"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java index 47a33d8a19e..f7648c86d4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java @@ -33,12 +33,9 @@ @Override public void cacheGroupsRefresh() throws IOException { - throw new UnsupportedOperationException(); } @Override public void cacheGroupsAdd(List groups) throws IOException { - throw new UnsupportedOperationException(); } - }