diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java index 78c34d9..ba13584 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.conf; import java.io.IOException; +import java.io.InputStream; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -53,6 +54,15 @@ public abstract Configuration getConfiguration(String name) throws YarnException, IOException; /** + * Get the configuration input stream + * @param name + * @return input stream of this configuration + * @throws YarnException + * @throws IOException + */ + public abstract InputStream getConfigurationInputStream(String name) + throws YarnException, IOException; + /** * Derived classes initialize themselves using this method. */ public abstract void initInternal(Configuration conf) throws Exception; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index 709f54a..bbcfa17 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn; import java.io.IOException; +import java.io.InputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,14 +45,20 @@ @Override public synchronized Configuration getConfiguration(String name) throws IOException, YarnException { + Configuration conf = new Configuration(false); + conf.addResource(getConfigurationInputStream(name)); + return conf; + } + + @Override + public synchronized InputStream getConfigurationInputStream(String name) + throws YarnException, IOException { Path configPath = new Path(this.configDir, name); if (!fs.exists(configPath)) { throw new YarnException("Can not find Configuration: " + name + " in " + configDir); } - Configuration conf = new Configuration(false); - conf.addResource(fs.open(configPath)); - return conf; + return fs.open(configPath); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index d152c35..8935536 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn; import java.io.IOException; +import java.io.InputStream; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -45,4 +46,11 @@ public void initInternal(Configuration conf) throws Exception { public void closeInternal() throws Exception { // Do nothing } + + @Override + public InputStream getConfigurationInputStream(String name) + throws YarnException, IOException { + // Do nothing + return null; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index d9c239e..693b8fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceStatus; @@ -45,11 +46,8 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; -import org.apache.hadoop.yarn.conf.ConfigurationProvider; -import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -92,8 +90,8 @@ private Server server; private InetSocketAddress masterServiceAddress; private AccessControlList adminAcl; - - private ConfigurationProvider configurationProvider = null; + + private boolean useLocalConfigurationProvider; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -115,9 +113,9 @@ public synchronized void serviceInit(Configuration conf) throws Exception { } } - this.configurationProvider = - ConfigurationProviderFactory.getConfigurationProvider(conf); - configurationProvider.init(conf); + this.useLocalConfigurationProvider = + this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider; masterServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, @@ -139,9 +137,6 @@ protected synchronized void serviceStart() throws Exception { @Override protected synchronized void serviceStop() throws Exception { stopServer(); - if (this.configurationProvider != null) { - configurationProvider.close(); - } super.serviceStop(); } @@ -158,7 +153,9 @@ protected void startServer() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + new RMPolicyProvider()); } if (rmContext.isHAEnabled()) { @@ -468,7 +465,7 @@ public RefreshServiceAclsResponse refreshServiceAcls( synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.configurationProvider instanceof LocalConfigurationProvider) { + if (this.useLocalConfigurationProvider) { this.server.refreshServiceAcl(configuration, policyProvider); } else { this.server.refreshServiceAclWithConfigration(configuration, @@ -523,7 +520,8 @@ public UpdateNodeResourceResponse updateNodeResource( private synchronized Configuration getConfiguration(String confFileName) throws YarnException, IOException { - return this.configurationProvider.getConfiguration(confFileName); + return this.rmContext.getConfigurationProvider().getConfiguration( + confFileName); } @VisibleForTesting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 2c4be13..bc1fdaa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -118,9 +118,8 @@ public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @Override protected void serviceInit(Configuration conf) throws Exception { this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); + this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider; super.serviceInit(conf); } @@ -150,7 +149,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + new RMPolicyProvider()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 086a6d8..0f23a9c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -154,9 +154,8 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); + this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider; super.serviceInit(conf); } @@ -175,7 +174,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + new RMPolicyProvider()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 64a4165..79fb5df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -97,4 +98,5 @@ void setRMDelegationTokenSecretManager( void setRMApplicationHistoryWriter( RMApplicationHistoryWriter rmApplicationHistoryWriter); + ConfigurationProvider getConfigurationProvider(); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 79e5983..cc52966 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; @@ -78,7 +79,7 @@ private ResourceTrackerService resourceTrackerService; private ApplicationMasterService applicationMasterService; private RMApplicationHistoryWriter rmApplicationHistoryWriter; - + private ConfigurationProvider configurationProvider; /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -120,7 +121,26 @@ public RMContextImpl(Dispatcher rmDispatcher, assert false; } } - + + @VisibleForTesting + // helper constructor for tests + public RMContextImpl(Dispatcher rmDispatcher, + ContainerAllocationExpirer containerAllocationExpirer, + AMLivelinessMonitor amLivelinessMonitor, + AMLivelinessMonitor amFinishingMonitor, + DelegationTokenRenewer delegationTokenRenewer, + AMRMTokenSecretManager appTokenSecretManager, + RMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInRM nmTokenSecretManager, + ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager, + RMApplicationHistoryWriter rmApplicationHistoryWriter, + ConfigurationProvider configurationProvider) { + this(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, + amFinishingMonitor, delegationTokenRenewer, appTokenSecretManager, + containerTokenSecretManager, nmTokenSecretManager, + clientToAMTokenSecretManager, rmApplicationHistoryWriter); + this.setConfigurationProvider(configurationProvider); + } @Override public Dispatcher getDispatcher() { return this.rmDispatcher; @@ -334,4 +354,13 @@ public void setRMApplicationHistoryWriter( this.rmApplicationHistoryWriter = rmApplicationHistoryWriter; } + @Override + public ConfigurationProvider getConfigurationProvider() { + return this.configurationProvider; + } + + public void setConfigurationProvider( + ConfigurationProvider configurationProvider) { + this.configurationProvider = configurationProvider; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 8575cd5..dc8072b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -42,10 +42,13 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -154,7 +157,7 @@ private boolean recoveryEnabled; private String webAppAddress; - + private ConfigurationProvider configurationProvider = null; /** End of Active services */ private Configuration conf; @@ -182,6 +185,19 @@ protected void serviceInit(Configuration conf) throws Exception { this.conf = conf; this.rmContext = new RMContextImpl(); + this.configurationProvider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + this.configurationProvider.init(this.conf); + rmContext.setConfigurationProvider(configurationProvider); + if (!(this.configurationProvider instanceof LocalConfigurationProvider)) { + this.conf.addResource(this.configurationProvider + .getConfigurationInputStream(YarnConfiguration.YARN_SITE_XML_FILE)); + // make sure core-site.xml is uploaded + this.conf.addResource(this.configurationProvider + .getConfigurationInputStream(YarnConfiguration + .CORE_SITE_CONFIGURATION_FILE)); + } + // register the handlers for all AlwaysOn services using setupDispatcher(). rmDispatcher = setupDispatcher(); addIfService(rmDispatcher); @@ -884,6 +900,9 @@ protected void serviceStop() throws Exception { if (fetcher != null) { fetcher.stop(); } + if (configurationProvider != null) { + configurationProvider.close(); + } super.serviceStop(); transitionToStandby(false); rmContext.setHAServiceState(HAServiceState.STOPPING); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4f74179..698288b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -146,9 +146,9 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); + this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider; + super.serviceInit(conf); } @@ -169,7 +169,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls(conf, new RMPolicyProvider()); + refreshServiceAcls( + this.rmContext.getConfigurationProvider().getConfiguration( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), + new RMPolicyProvider()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index b019a76..a56687b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; @@ -263,10 +264,8 @@ public Resource getClusterResources() { public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.useLocalConfigurationProvider = - (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - LocalConfigurationProvider.class))); + this.useLocalConfigurationProvider = rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider; this.conf = new CapacitySchedulerConfiguration(conf, this.useLocalConfigurationProvider); @@ -279,6 +278,18 @@ public Resource getClusterResources() { new ConcurrentHashMap(); this.rmContext = rmContext; + if (! this.useLocalConfigurationProvider) { + try { + this.conf = + new CapacitySchedulerConfiguration(rmContext + .getConfigurationProvider().getConfiguration( + YarnConfiguration.CS_CONFIGURATION_FILE), + this.useLocalConfigurationProvider); + + } catch (YarnException ex) { + throw new IOException(ex); + } + } initializeQueues(this.conf); initialized = true; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 5372c18..ee008e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -26,7 +26,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.authorize.AccessControlList; @@ -105,34 +104,34 @@ public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } + + //upload default configurations + uploadDefaultConfiguration(); - // clean the remoteDirectory - cleanRemoteDirectory(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); + } CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); - try { - rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: capacity-scheduler.xml")); - } - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); - String csConfFile = writeConfigurationXML(csConf, - "capacity-scheduler.xml"); - - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(csConfFile)); + uploadConfiguration(csConf, "capacity-scheduler.xml"); rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); @@ -159,20 +158,24 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: yarn-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } String aclStringBefore = @@ -180,10 +183,8 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider() YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); - String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + uploadConfiguration(yarnConf, "yarn-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(yarnConfFile)); rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); String aclStringAfter = @@ -214,7 +215,6 @@ public void testServiceAclsRefreshWithLocalConfigurationProvider() { } } - @SuppressWarnings("resource") @Test public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { @@ -224,33 +224,33 @@ public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); ResourceManager resourceManager = null; try { - resourceManager = new ResourceManager(); - resourceManager.init(configuration); - resourceManager.start(); - - // clean the remoteDirectory - cleanRemoteDirectory(); - try { - resourceManager.adminService - .refreshServiceAcls(RefreshServiceAclsRequest - .newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + fail("Should throw an exception"); } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: hadoop-policy.xml")); + // expect to get an exception here } - String aclsString = "alice,bob users,wheel"; + //upload default configurations + uploadDefaultConfiguration(); Configuration conf = new Configuration(); conf.setBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); - conf.set("security.applicationclient.protocol.acl", aclsString); - String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + uploadConfiguration(conf, "core-site.xml"); + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(hadoopConfFile)); + String aclsString = "alice,bob users,wheel"; + Configuration newConf = new Configuration(); + newConf.set("security.applicationclient.protocol.acl", aclsString); + uploadConfiguration(newConf, "hadoop-policy.xml"); resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest .newInstance()); @@ -328,31 +328,31 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshSuperUserGroupsConfiguration( - RefreshSuperUserGroupsConfigurationRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: core-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); - String coreConfFile = writeConfigurationXML(coreConf, - "core-site.xml"); + uploadConfiguration(coreConf, "core-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(coreConfFile)); rm.adminService.refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest.newInstance()); Assert.assertTrue(ProxyUsers.getProxyGroups() @@ -393,11 +393,29 @@ private void uploadToRemoteFileSystem(Path filePath) fs.copyFromLocalFile(filePath, workingPath); } - private void cleanRemoteDirectory() throws IOException { - if (fs.exists(workingPath)) { - for (FileStatus file : fs.listStatus(workingPath)) { - fs.delete(file.getPath(), true); - } - } + private void uploadConfiguration(Configuration conf, String confFileName) + throws IOException { + String csConfFile = writeConfigurationXML(conf, confFileName); + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + } + + private void uploadDefaultConfiguration() throws IOException { + Configuration conf = new Configuration(); + uploadConfiguration(conf, "core-site.xml"); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + uploadConfiguration(csConf, "capacity-scheduler.xml"); + + Configuration hadoopPolicyConf = new Configuration(false); + hadoopPolicyConf + .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml"); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ca60db3..d8765c6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -132,8 +133,11 @@ public void testConfValidation() throws Exception { Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + RMContext mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { @@ -147,7 +151,7 @@ public void testConfValidation() throws Exception { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation."); } catch (YarnRuntimeException e) { @@ -348,7 +352,8 @@ public void testRefreshQueues() throws Exception { cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); @@ -447,7 +452,8 @@ public void testParseQueue() throws IOException { cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); } @Test @@ -460,7 +466,8 @@ public void testReconnectedNode() throws Exception { cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); @@ -487,7 +494,8 @@ public void testRefreshQueuesWithNewQueue() throws Exception { cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); // Add a new queue b4 @@ -638,7 +646,8 @@ public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { cs.reinitialize(conf, new RMContextImpl(rmDispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); SchedulerApplication app = TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 21c446a..28e29cb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -47,7 +48,8 @@ public void testQueueParsing() throws Exception { capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); CSQueue a = capacityScheduler.getQueue("a"); Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index db28dca..1ee1569 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -91,7 +92,8 @@ public EventHandler getEventHandler() { new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), writer); + new ClientToAMTokenSecretManagerInRM(), writer, + new LocalConfigurationProvider()); return rmContext; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 2c2aae6..cbfa7e4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -206,7 +207,8 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null, + new LocalConfigurationProvider())); return cs; }