diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index 097bc30..ea18b94 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -27,6 +27,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -240,4 +241,18 @@ public static synchronized Groups getUserToGroupsMappingService( } return GROUPS; } + + /** + * Create new groups used to map user-to-groups with loaded configuration. + * @param conf + * @return the groups being used to map user-to-groups. + */ + @Private + public static synchronized Groups + getUserToGroupsMappingServiceWithLoadedConfiguration( + Configuration conf) { + + GROUPS = new Groups(conf); + return GROUPS; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index da479b4..6ebf90a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -381,21 +381,22 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu @Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshUserToGroupsMappings"); + throws YarnException, IOException { + String argName = "refreshUserToGroupsMappings"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshUserToGroupsMapping", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh user-groups."); throwStandbyException(); } - Groups.getUserToGroupsMappingService().refresh(); - RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshUserToGroupsMappings", "AdminService"); + Groups.getUserToGroupsMappingService( + getConfiguration(getConfig(), + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); + + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 1040cc5..5ef58a7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; @@ -198,6 +199,9 @@ protected void serviceInit(Configuration conf) throws Exception { this.conf = this.configurationProvider.getConfiguration(this.conf, YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + // Do refreshUserToGroupsMappings with loaded core-site.xml + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf) + .refresh(); } // register the handlers for all AlwaysOn services using setupDispatcher(). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index ee008e9..9746664 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,10 +24,17 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; @@ -37,6 +44,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -366,6 +374,84 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } + @Test + public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + try { + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch (Exception ex) { + // Expect exception here + } + + String user = UserGroupInformation.getCurrentUser().getUserName(); + List groupWithInit = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + + // upload default configurations + uploadDefaultConfiguration(); + Configuration conf = new Configuration(); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + MockUnixGroupsMapping.class, + GroupMappingServiceProvider.class); + uploadConfiguration(conf, "core-site.xml"); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + // Make sure RM will use the updated GroupMappingServiceProvider + List groupBefore = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + Assert.assertTrue(groupBefore.contains("test_group_A") + && groupBefore.contains("test_group_B") + && groupBefore.contains("test_group_C") && groupBefore.size() == 3); + Assert.assertTrue(groupWithInit.size() != groupBefore.size()); + Assert.assertFalse(groupWithInit.contains("test_group_A") + || groupWithInit.contains("test_group_B") + || groupWithInit.contains("test_group_C")); + + // update the groups + MockUnixGroupsMapping.updateGroups(); + + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + List groupAfter = + Groups.getUserToGroupsMappingService(configuration).getGroups(user); + + // should get the updated groups + Assert.assertTrue(groupAfter.contains("test_group_D") + && groupAfter.contains("test_group_E") + && groupAfter.contains("test_group_F") && groupAfter.size() == 3); + + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; @@ -418,4 +504,38 @@ private void uploadDefaultConfiguration() throws IOException { .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml"); } + + private static class MockUnixGroupsMapping implements + GroupMappingServiceProvider { + + @SuppressWarnings("serial") + private static List group = new ArrayList() {{ + add("test_group_A"); + add("test_group_B"); + add("test_group_C"); + }}; + + @Override + public List getGroups(String user) throws IOException { + return group; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + // Do nothing + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + // Do nothing + } + + public static void updateGroups() { + group.clear(); + group.add("test_group_D"); + group.add("test_group_E"); + group.add("test_group_F"); + } + } + }