diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0bd893a..cfc1693 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -38,6 +38,7 @@ public class YarnConfiguration extends Configuration { public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index b568c25..bcb4d66 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,7 +40,11 @@ import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.HAUtil; @@ -48,6 +53,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -337,6 +343,81 @@ public void testAdminRefreshQueuesWithRemoteConfigurationSwitchOn() Assert.assertEquals(maxAppsAfter, 5000); Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + + @Test + public void testAdminRefreshSuperUserGroupsWithRemoteConfigurationSwitchOff() + throws IOException { + Configuration conf = new Configuration(this.conf); + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + RefreshSuperUserGroupsConfigurationRequest request = + RefreshSuperUserGroupsConfigurationRequest.newInstance(); + try { + getAdminService(0).refreshSuperUserGroupsConfiguration(request); + } catch (Exception ex) { + fail("The HA is enabled, but RM_HA_REMOTE_CONFIGURATION_ENABLED" + + " is set as false. Should not get any exception" + ex.getMessage()); + } + } + + @Test + public void testAdminRefreshSuperUserGroupsWithRemoteConfigurationSwitchOn() + throws IOException, YarnException { + final String REAL_USER_NAME = "proxier"; + final String PROXY_USER_NAME = "proxied_user"; + final String[] GROUP_NAMES = new String[] { "test_group" }; + final String PROXY_IP = "1.2.3.4"; + // switch RM_HA_REMOTE_CONFIGURATION_ENABLED to true + conf.setBoolean(YarnConfiguration.RM_HA_REMOTE_CONFIGURATION_ENABLED + , true); + Configuration conf = new Configuration(this.conf); + conf.set( + ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + StringUtils.join(",", Arrays.asList(GROUP_NAMES))); + conf.set( + ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + PROXY_IP); + UserGroupInformation realUserUgi = UserGroupInformation + .createRemoteUser(REAL_USER_NAME); + UserGroupInformation proxyUserUgi = + UserGroupInformation.createProxyUserForTesting( + PROXY_USER_NAME, realUserUgi, GROUP_NAMES); + + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + RefreshSuperUserGroupsConfigurationRequest request = + RefreshSuperUserGroupsConfigurationRequest.newInstance(); + try { + getAdminService(0).refreshSuperUserGroupsConfiguration(request); + fail("The HA is enabled. The remote configuration has not been set." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: core-site.xml")); + } + + String coreConfFile = writeConfigurationXML(conf, "core-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(coreConfFile)); + + getAdminService(0).refreshSuperUserGroupsConfiguration(request); + + assertAuthorized(proxyUserUgi, PROXY_IP); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; @@ -375,4 +456,12 @@ private void cleanRemoteDirectory() throws IOException { } } } + + private void assertAuthorized(UserGroupInformation proxyUgi, String host) { + try { + ProxyUsers.authorize(proxyUgi, host, null); + } catch (AuthorizationException e) { + fail("Did not allowed authorization of " + proxyUgi + " from " + host); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 10d6856..ff8b1c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -367,19 +367,20 @@ public RefreshNodesResponse refreshNodes(RefreshNodesRequest request) @Override public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration"); + throws YarnException, IOException { + String argName = "refreshSuperUserGroupsConfiguration"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh super-user-groups."); throwStandbyException(); } - ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration()); + Configuration conf = + getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), "refreshSuperUserGroupsConfiguration", "AdminService");