diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0bd893a..cfc1693 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -38,6 +38,7 @@ public class YarnConfiguration extends Configuration { public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index b568c25..016c533 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -29,6 +29,8 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,6 +41,9 @@ import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -48,6 +53,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -109,6 +115,11 @@ public void setup() throws IOException { conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); + conf.setClass("hadoop.security.group.mapping", + MockUnixGroupsMapping.class, + GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingService(conf); + fs = FileSystem.get(conf); workingPath = new Path(fs.getHomeDirectory(), "confStore"); conf.set(YarnConfiguration.FS_RM_CONF_STORE, workingPath.toString()); @@ -337,6 +348,67 @@ public void testAdminRefreshQueuesWithRemoteConfigurationSwitchOn() Assert.assertEquals(maxAppsAfter, 5000); Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + + @Test + public void testRefreshUserToGroupsMappingsWithRemoteConfigurationSwitchOff() + throws IOException, YarnException { + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + getAdminService(0).refreshUserToGroupsMappings( + RefreshUserToGroupsMappingsRequest.newInstance()); + } catch (Exception ex) { + fail("The HA is enabled, but RM_HA_REMOTE_CONFIGURATION_ENABLED" + + " is set as false. Should not get any exception" + ex.getMessage()); + } + } + + @Test + public void testRefreshUserToGroupsMappingsWithRemoteConfigurationSwitchOn() + throws IOException, YarnException { + // switch RM_HA_REMOTE_CONFIGURATION_ENABLED to true + conf.setBoolean(YarnConfiguration.RM_HA_REMOTE_CONFIGURATION_ENABLED, true); + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + RefreshUserToGroupsMappingsRequest request = + RefreshUserToGroupsMappingsRequest.newInstance(); + try { + getAdminService(0).refreshUserToGroupsMappings(request); + fail("The HA is enabled. The remote configuration has not been set." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: core-site.xml")); + } + Groups groups = Groups.getUserToGroupsMappingService(conf); + String user = UserGroupInformation.getCurrentUser().getUserName(); + List groupBefore = groups.getGroups(user); + + String coreConfFile = writeConfigurationXML(conf, "core-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(coreConfFile)); + getAdminService(0).refreshUserToGroupsMappings(request); + List groupAfter = groups.getGroups(user); + + for (int i = 0; i < groupAfter.size(); i++) { + assertFalse("Should be different group: " + groupBefore.get(i) + " and " + + groupAfter.get(i), groupBefore.get(i).equals(groupAfter.get(i))); + } + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; @@ -375,4 +447,31 @@ private void cleanRemoteDirectory() throws IOException { } } } + + public static class MockUnixGroupsMapping implements + GroupMappingServiceProvider { + + private int i = 0; + + @Override + public List getGroups(String user) throws IOException { + System.out.println("Getting groups in MockUnixGroupsMapping"); + String g1 = user + (10 * i + 1); + String g2 = user + (10 * i + 2); + List l = new ArrayList(2); + l.add(g1); + l.add(g2); + i++; + return l; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + System.out.println("Refreshing groups in MockUnixGroupsMapping"); + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 10d6856..996b489 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -390,21 +390,26 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu @Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshUserToGroupsMappings"); + throws YarnException, IOException { + String argName = "refreshUserToGroupsMappings"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling user-groups on Standby RM if (!isRMActive()) { RMAuditLogger.logFailure(user.getShortUserName(), - "refreshUserToGroupsMapping", - adminAcl.toString(), "AdminService", + argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh user-groups."); throwStandbyException(); } - Groups.getUserToGroupsMappingService().refresh(); - RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshUserToGroupsMappings", "AdminService"); + Configuration conf = + getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + if (this.rmContext.isHAEnabled() && this.remoteConfigurationEnabled) { + Groups.getUserToGroupsMappingService(conf).refresh(); + } else { + Groups.getUserToGroupsMappingService().refresh(); + RMAuditLogger.logSuccess(user.getShortUserName(), + "refreshUserToGroupsMappings", "AdminService"); + } return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class);