diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index b4f97123c0f..f0ab7c74b47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -18,12 +18,19 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; +import org.apache.hadoop.yarn.util.Records; +import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; @@ -125,4 +132,93 @@ public void testUpdatePlacementRulesFactory() throws Exception { assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_USER_GROUP)); assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_APP_NAME)); } + + @Test + public void testDynamicPrimaryGroupQueue() throws Exception { + + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + + List queuePlacementRules = new ArrayList<>(); + queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); + conf.setQueuePlacementRules(queuePlacementRules); + + List existingMappingsForUG = + conf.getQueueMappings(); + + // set queue mapping + List queueMappingsForUG = + new ArrayList<>(); + + // u:user1:b1 + UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user1", + "b1"); + + // u:user2:%primary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user2", + "%primary_group"); + + // u:%user:%primary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", + "%primary_group"); + + // u:%user:parentqueue.%user + UserGroupMappingPlacementRule.QueueMapping userQueueMapping4 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", + getQueueMapping("c", "%user")); + queueMappingsForUG.add(userQueueMapping1); + queueMappingsForUG.add(userQueueMapping2); + queueMappingsForUG.add(userQueueMapping3); + queueMappingsForUG.add(userQueueMapping4); + + existingMappingsForUG.addAll(queueMappingsForUG); + conf.setQueueMappings(existingMappingsForUG); + + //override with queue mappings + conf.setOverrideWithQueueMappings(true); + + mockRM = new MockRM(conf); + CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler(); + cs.updatePlacementRules(); + mockRM.start(); + cs.start(); + + ApplicationSubmissionContext asc = Records.newRecord(ApplicationSubmissionContext.class); + asc.setQueue("default"); + + List rules = cs.getRMContext().getQueuePlacementManager().getPlacementRules(); + + UserGroupMappingPlacementRule r = (UserGroupMappingPlacementRule) rules.get(0); + + ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1"); + Assert.assertEquals("b1", ctx != null ? ctx.getQueue() : "default"); + + // ctx1.getQueue() is supposed to have "user2group" + ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2"); + Assert.assertEquals("%primary_group", + ctx1 != null ? ctx1.getQueue() : "default"); + + ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user3"); + Assert.assertEquals("user3group", + ctx2 != null ? ctx2.getQueue() : "default"); + + // ctx1.getQueue() is supposed to have "user4" + // ctx1.getParentQueue() is supposed to have "c" + ApplicationPlacementContext ctx3 = r.getPlacementForApp(asc, "user4"); + Assert.assertEquals("user4group", + ctx3 != null ? ctx3.getQueue() : "default"); + Assert.assertEquals(null, + ctx3 != null ? ctx3.getParentQueue() : "default"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java index 47a33d8a19e..a683b2f3dd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java @@ -33,12 +33,10 @@ @Override public void cacheGroupsRefresh() throws IOException { - throw new UnsupportedOperationException(); } @Override public void cacheGroupsAdd(List groups) throws IOException { - throw new UnsupportedOperationException(); } }