diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index c70527f3012..c457b2d5281 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -313,9 +313,9 @@ public boolean initialize(ResourceScheduler scheduler) // initialize groups if mappings are present if (newMappings.size() > 0) { - Groups groups = new Groups(conf); this.mappings = newMappings; - this.groups = groups; + this.groups = Groups.getUserToGroupsMappingService( + ((CapacityScheduler)scheduler).getConfig()); this.overrideWithQueueMappings = overrideWithQueueMappings; return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java index 30ea213529d..ecbd170925e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -72,7 +72,7 @@ public QueuePlacementPolicy(List rules, } this.rules = rules; this.configuredQueues = configuredQueues; - groups = new Groups(conf); + this.groups = Groups.getUserToGroupsMappingService(conf); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 7ef8b9eae4f..5572960be95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -49,6 +49,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.security.UserGroupInformation; @@ -4855,6 +4856,7 @@ public void testQueueMappingWithCurrentUserQueueMappingForaGroup() throws "a1" +"=" + "agroup" + ""); config.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "g:agroup:%user"); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(config); MockRM rm = new MockRM(config); rm.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index c64582ef21f..97d854267ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -23,7 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -63,8 +63,6 @@ .AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .SimpleGroupsMapping; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; @@ -296,6 +294,7 @@ public static CapacitySchedulerConfiguration setupQueueMappings( TestGroupsCaching.FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class); conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); UserGroupMappingPlacementRule.QueueMapping userQueueMapping = new UserGroupMappingPlacementRule.QueueMapping( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..afa9797ead9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -54,6 +54,8 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1916,6 +1918,7 @@ public void testAssignToNonLeafQueueReturnsNull() throws Exception { public void testQueuePlacementWithPolicy() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1961,6 +1964,11 @@ public void testQueuePlacementWithPolicy() throws Exception { assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "otheruser"); assertEquals("root.default", scheduler.getSchedulerApp(appId).getQueueName()); + + // undo the group change + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + JniBasedUnixGroupsMappingWithFallback.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test @@ -2013,6 +2021,7 @@ public void testNestedUserQueue() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); @@ -2038,6 +2047,10 @@ public void testNestedUserQueue() throws IOException { "user1"); assertEquals("root.user1group.user1", user1Leaf.getName()); + // undo the group change + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + JniBasedUnixGroupsMappingWithFallback.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 3fe9ce31021..47d58b3da7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -20,10 +20,8 @@ import static org.junit.Assert.*; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -389,8 +388,10 @@ public void testGroupContainsPeriod() throws Exception { sb.append(" "); sb.append(""); + // change the group resolution conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, PeriodGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); // User queue would be created under primary group queue, and the period // in the group name should be converted into _dot_ QueuePlacementPolicy policy = parse(sb.toString()); @@ -399,6 +400,7 @@ public void testGroupContainsPeriod() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test(expected=IOException.class) @@ -412,6 +414,7 @@ public void testEmptyGroupsPrimaryGroupRule() throws Exception { // Add a static mapping that returns empty groups for users conf.setStrings(CommonConfigurationKeys .HADOOP_USER_GROUP_STATIC_OVERRIDES, "emptygroupuser="); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); QueuePlacementPolicy policy = parse(sb.toString()); policy.assignAppToQueue(null, "emptygroupuser"); }