diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index c7cada8..698280a 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -451,6 +451,14 @@ public void refreshServiceAcl(Configuration conf, PolicyProvider provider) { } /** + * Refresh the service authorization ACL for the service handled by this server + * using the specified Configuration. + */ + public void refreshServiceAclWtihConfigration(Configuration conf, + PolicyProvider provider) { + serviceAuthorizationManager.refreshWithConfiguration(conf, provider); + } + /** * Returns a handle to the serviceAuthorizationManager (required in tests) * @return instance of ServiceAuthorizationManager for this server */ diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 8523f38..96a86a1 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,6 +33,8 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import com.google.common.annotations.VisibleForTesting; + /** * An authorization manager which handles service-level authorization * for incoming service requests. @@ -120,19 +122,28 @@ public synchronized void refresh(Configuration conf, // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); - + refreshProtocolToAcls(policyConf, provider); + } + + public synchronized void refreshWithConfiguration(Configuration conf, + PolicyProvider provider) { + refreshProtocolToAcls(conf, provider); + } + + private void refreshProtocolToAcls(Configuration conf, + PolicyProvider provider) { final Map, AccessControlList> newAcls = - new IdentityHashMap, AccessControlList>(); + new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { - AccessControlList acl = - new AccessControlList( - policyConf.get(service.getServiceKey(), - AccessControlList.WILDCARD_ACL_VALUE) - ); + AccessControlList acl = + new AccessControlList( + conf.get(service.getServiceKey(), + AccessControlList.WILDCARD_ACL_VALUE) + ); newAcls.put(service.getProtocol(), acl); } } @@ -141,8 +152,13 @@ public synchronized void refresh(Configuration conf, protocolToAcl = newAcls; } - // Package-protected for use in tests. - Set> getProtocolsWithAcls() { + @VisibleForTesting + public Set> getProtocolsWithAcls() { return protocolToAcl.keySet(); } + + @VisibleForTesting + public AccessControlList getProtocolsAcls(Class className) { + return protocolToAcl.get(className); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 919ed90..44f6e9b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -41,6 +41,10 @@ public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; @Private + public static final String HADOOP_POLICY_CONFIGURATION_FILE = + "hadoop-policy.xml"; + + @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; @Private diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3bfd47d..08cd7be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.conf.ConfigurationProvider; @@ -432,9 +433,8 @@ public RefreshAdminAclsResponse refreshAdminAcls( @Override public RefreshServiceAclsResponse refreshServiceAcls( - RefreshServiceAclsRequest request) throws YarnException { - Configuration conf = new Configuration(); - if (!conf.getBoolean( + RefreshServiceAclsRequest request) throws YarnException, IOException { + if (!getConfig().getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { throw RPCUtil.getRemoteException( @@ -442,27 +442,38 @@ public RefreshServiceAclsResponse refreshServiceAcls( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION + ") not enabled.")); } - + + String argName = "refreshServiceAcls"; + if (!isRMActive()) { + RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser() + .getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh Service ACLs."); + throwStandbyException(); + } + PolicyProvider policyProvider = new RMPolicyProvider(); - + Configuration conf = + getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + refreshServiceAcls(conf, policyProvider); - if (isRMActive()) { - rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); - rmContext.getApplicationMasterService().refreshServiceAcls( - conf, policyProvider); - rmContext.getResourceTrackerService().refreshServiceAcls( - conf, policyProvider); - } else { - LOG.warn("ResourceManager is not active. Not refreshing ACLs for " + - "Clients, ApplicationMasters and NodeManagers"); - } + rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); + rmContext.getApplicationMasterService().refreshServiceAcls( + conf, policyProvider); + rmContext.getResourceTrackerService().refreshServiceAcls( + conf, policyProvider); return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); } void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.configurationProvider instanceof LocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWtihConfigration(configuration, + policyProvider); + } } @Override @@ -519,4 +530,9 @@ private synchronized Configuration getConfiguration(String confFileName) public AccessControlList getAccessControlList() { return this.adminAcl; } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 57605c0..20a47b2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -102,6 +102,7 @@ private final AllocateResponse resync = recordFactory.newRecordInstance(AllocateResponse.class); private final RMContext rmContext; + private boolean useLocalConfigurationProvider; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); @@ -112,6 +113,15 @@ public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { } @Override + protected void serviceInit(Configuration conf) throws Exception { + this.useLocalConfigurationProvider = conf.get( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( + "org.apache.hadoop.yarn.LocalConfigurationProvider"); + super.serviceInit(conf); + } + + @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); @@ -578,7 +588,12 @@ public void unregisterAttempt(ApplicationAttemptId attemptId) { public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWtihConfigration(configuration, + policyProvider); + } } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 8800f29..0704096 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -133,6 +133,7 @@ private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; + private boolean useLocalConfigurationProvider; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -150,6 +151,10 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); + this.useLocalConfigurationProvider = conf.get( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( + "org.apache.hadoop.yarn.LocalConfigurationProvider"); super.serviceInit(conf); } @@ -773,7 +778,12 @@ private String getRenewerForToken(Token token) void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWtihConfigration(configuration, + policyProvider); + } } private boolean isAllowedDelegationTokenOp() throws IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f80ce85..dd2a0fc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -92,6 +92,7 @@ private int minAllocMb; private int minAllocVcores; + private boolean useLocalConfigurationProvider; static { resync.setNodeAction(NodeAction.RESYNC); @@ -141,6 +142,10 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + this.useLocalConfigurationProvider = conf.get( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( + "org.apache.hadoop.yarn.LocalConfigurationProvider"); super.serviceInit(conf); } @@ -415,6 +420,11 @@ public static Node resolve(String hostName) { void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWtihConfigration(configuration, + policyProvider); + } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 797b422..f87cdc0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,16 +24,19 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -191,6 +194,89 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider() } @Test + public void testServiceAclsRefreshWithLocalConfigurationProvider() { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + ResourceManager resourceManager = null; + + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + @SuppressWarnings("resource") + @Test + public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + ResourceManager resourceManager = null; + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + resourceManager.adminService + .refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: hadoop-policy.xml")); + } + + Configuration conf = new Configuration(); + conf.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + conf.set("security.applicationclient.protocol.acl", + "alice,bob users,wheel"); + String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(hadoopConfFile)); + + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + + ServiceAuthorizationManager adminServiceServiceManager = + resourceManager.adminService.getServer() + .getServiceAuthorizationManager(); + for (Class protocolClass : adminServiceServiceManager + .getProtocolsWithAcls()) { + AccessControlList accessList = + adminServiceServiceManager.getProtocolsAcls(protocolClass); + if (protocolClass == org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class) { + Assert.assertEquals(accessList.getAclString(), + "alice,bob users,wheel"); + } else { + Assert.assertEquals(accessList.getAclString(), "*"); + } + } + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + @Test public void testRefreshSuperUserGroupsWithLocalConfigurationProvider() { rm = new MockRM(configuration);