diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java index d557309..cd24b4a 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; // Keeps track of which datanodes/tasktrackers are allowed to connect to the @@ -48,13 +49,30 @@ public HostsFileReader(String inFile, refresh(); } + @Private + public HostsFileReader(String inFile, InputStream inFileInputStream, + String exFile, InputStream exFileInputStream) throws IOException { + includes = new HashSet(); + excludes = new HashSet(); + includesFile = inFile; + excludesFile = exFile; + refresh(inFileInputStream, exFileInputStream); + } + public static void readFileToSet(String type, String filename, Set set) throws IOException { File file = new File(filename); FileInputStream fis = new FileInputStream(file); + readFileToSetWithFileInputStream(type, filename, fis, set); + } + + @Private + public static void readFileToSetWithFileInputStream(String type, + String filename, InputStream fileInputStream, Set set) + throws IOException { BufferedReader reader = null; try { - reader = new BufferedReader(new InputStreamReader(fis)); + reader = new BufferedReader(new InputStreamReader(fileInputStream)); String line; while ((line = reader.readLine()) != null) { String[] nodes = line.split("[ \t\n\f\r]+"); @@ -71,13 +89,13 @@ public static void readFileToSet(String type, } } } - } + } } finally { if (reader != null) { reader.close(); } - fis.close(); - } + fileInputStream.close(); + } } public synchronized void refresh() throws IOException { @@ -96,6 +114,26 @@ public synchronized void refresh() throws IOException { } } + @Private + public synchronized void refresh(InputStream inFileInputStream, + InputStream exFileInputStream) throws IOException { + LOG.info("Refreshing hosts (include/exclude) list"); + if (!includesFile.isEmpty() && inFileInputStream != null) { + Set newIncludes = new HashSet(); + readFileToSetWithFileInputStream("included", includesFile, + inFileInputStream, newIncludes); + // switch the new hosts that are to be included + includes = newIncludes; + } + if (!excludesFile.isEmpty() && exFileInputStream != null) { + Set newExcludes = new HashSet(); + readFileToSetWithFileInputStream("excluded", excludesFile, + exFileInputStream, newExcludes); + // switch the excluded hosts + excludes = newExcludes; + } + } + public synchronized Set getHosts() { return includes; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java index b31573d..df8dd4e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.conf; import java.io.IOException; +import java.io.InputStream; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -42,14 +44,13 @@ public void close() throws Exception { } /** - * Get the configuration and combine with bootstrapConf - * @param bootstrapConf Configuration + * Opens an InputStream at the indicated file * @param name The configuration file name * @return configuration * @throws YarnException * @throws IOException */ - public abstract Configuration getConfiguration(Configuration bootstrapConf, + public abstract InputStream getConfigurationInputStream( String name) throws YarnException, IOException; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8c8ad16..8265b1a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -52,6 +52,14 @@ @Private public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + @Private + public static final List RM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_XML_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index 390aace..0d56bd4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn; import java.io.IOException; +import java.io.InputStream; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -41,15 +43,25 @@ private Path configDir; @Override - public synchronized Configuration getConfiguration(Configuration bootstrapConf, - String name) throws IOException, YarnException { - Path configPath = new Path(this.configDir, name); - if (!fs.exists(configPath)) { - throw new YarnException("Can not find Configuration: " + name + " in " - + configDir); + public synchronized InputStream getConfigurationInputStream(String name) + throws IOException, YarnException { + if (name == null || name.isEmpty()) { + return null; + } + Path filePath; + if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + filePath = new Path(this.configDir, name); + if (!fs.exists(filePath)) { + throw new YarnException("Can not find Configuration: " + name + " in " + + configDir); + } + } else { + filePath = new Path(name); + if (!fs.exists(filePath)) { + throw new YarnException("Can not find file: " + name); + } } - bootstrapConf.addResource(fs.open(configPath)); - return bootstrapConf; + return fs.open(filePath); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index 3e69960..bfbb39a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -18,11 +18,15 @@ package org.apache.hadoop.yarn; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @Private @@ -30,9 +34,13 @@ public class LocalConfigurationProvider extends ConfigurationProvider { @Override - public Configuration getConfiguration(Configuration bootstrapConf, - String name) throws IOException, YarnException { - return bootstrapConf; + public InputStream getConfigurationInputStream(String name) + throws IOException, YarnException { + if (name == null || name.isEmpty() + || YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + return null; + } + return new FileInputStream(name); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index da479b4..b26e7a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -331,23 +331,27 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) @Override public RefreshNodesResponse refreshNodes(RefreshNodesRequest request) throws YarnException, StandbyException { + String argName = "refreshNodes"; UserGroupInformation user = checkAcls("refreshNodes"); if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh nodes."); throwStandbyException(); } try { - rmContext.getNodesListManager().refreshNodes(new YarnConfiguration()); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshNodes", + Configuration conf = + getConfiguration(new YarnConfiguration(), + YarnConfiguration.YARN_SITE_XML_FILE); + rmContext.getNodesListManager().refreshNodes(conf); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshNodesResponse.class); } catch (IOException ioe) { LOG.info("Exception refreshing nodes ", ioe); - RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "Exception refreshing nodes"); throw RPCUtil.getRemoteException(ioe); } @@ -368,7 +372,7 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu } Configuration conf = - getConfiguration(getConfig(), + getConfiguration(new Configuration(), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), @@ -414,7 +418,8 @@ public RefreshAdminAclsResponse refreshAdminAcls( throwStandbyException(); } Configuration conf = - getConfiguration(getConfig(), YarnConfiguration.YARN_SITE_XML_FILE); + getConfiguration(new Configuration(), + YarnConfiguration.YARN_SITE_XML_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); @@ -447,7 +452,7 @@ public RefreshServiceAclsResponse refreshServiceAcls( PolicyProvider policyProvider = RMPolicyProvider.getInstance(); Configuration conf = - getConfiguration(getConfig(), + getConfiguration(new Configuration(), YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); refreshServiceAcls(conf, policyProvider); @@ -518,8 +523,12 @@ public UpdateNodeResourceResponse updateNodeResource( private synchronized Configuration getConfiguration(Configuration conf, String confFileName) throws YarnException, IOException { - return this.rmContext.getConfigurationProvider().getConfiguration(conf, - confFileName); + if (!(this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider)) { + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(confFileName)); + } + return conf; } @VisibleForTesting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 0c56134..68fcf3d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -140,10 +140,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + if (!(this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider)) { + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationInputStream( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + } + refreshServiceAcls(conf,RMPolicyProvider.getInstance()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 43e94ed..30d89d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -171,10 +171,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + if (!(this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider)) { + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationInputStream( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + } + refreshServiceAcls(conf,RMPolicyProvider.getInstance()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index 4249980..38d9122 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -32,12 +32,15 @@ import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import com.google.common.annotations.VisibleForTesting; + @SuppressWarnings("unchecked") public class NodesListManager extends AbstractService implements EventHandler { @@ -63,25 +66,21 @@ protected void serviceInit(Configuration conf) throws Exception { // Read the hosts/exclude files to restrict access to the RM try { - this.hostsReader = - new HostsFileReader( - conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), - conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH) - ); + String inFile = conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); + String exFile = conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); + this.hostsReader = + new HostsFileReader(inFile, + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(inFile), exFile, + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(exFile)); printConfiguredHosts(); + } catch (YarnException ex) { + disableHostsFileReader(ex); } catch (IOException ioe) { - LOG.warn("Failed to init hostsReader, disabling", ioe); - try { - this.hostsReader = - new HostsFileReader(YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); - } catch (IOException ioe2) { - // Should *never* happen - this.hostsReader = null; - throw new YarnRuntimeException(ioe2); - } + disableHostsFileReader(ioe); } super.serviceInit(conf); } @@ -103,7 +102,8 @@ private void printConfiguredHosts() { } } - public void refreshNodes(Configuration yarnConf) throws IOException { + public void refreshNodes(Configuration yarnConf) throws IOException, + YarnException { synchronized (hostsReader) { if (null == yarnConf) { yarnConf = new YarnConfiguration(); @@ -113,7 +113,15 @@ public void refreshNodes(Configuration yarnConf) throws IOException { YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), yarnConf.get( YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); - hostsReader.refresh(); + String inFile = + yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); + String exFile = + yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); + hostsReader.refresh(this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(inFile), this.rmContext + .getConfigurationProvider().getConfigurationInputStream(exFile)); printConfiguredHosts(); } } @@ -174,4 +182,23 @@ public void handle(NodesListManagerEvent event) { LOG.error("Ignoring invalid eventtype " + event.getType()); } } + + private void disableHostsFileReader(Exception ex) { + LOG.warn("Failed to init hostsReader, disabling", ex); + try { + this.hostsReader = + new HostsFileReader( + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); + } catch (IOException ioe2) { + // Should *never* happen + this.hostsReader = null; + throw new YarnRuntimeException(ioe2); + } + } + + @VisibleForTesting + public HostsFileReader getHostsReader() { + return this.hostsReader; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 1040cc5..ef351e7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -191,13 +191,12 @@ protected void serviceInit(Configuration conf) throws Exception { rmContext.setConfigurationProvider(configurationProvider); if (!(this.configurationProvider instanceof LocalConfigurationProvider)) { // load yarn-site.xml - this.conf = - this.configurationProvider.getConfiguration(this.conf, - YarnConfiguration.YARN_SITE_XML_FILE); + this.conf.addResource(this.configurationProvider + .getConfigurationInputStream(YarnConfiguration.YARN_SITE_XML_FILE)); // load core-site.xml - this.conf = - this.configurationProvider.getConfiguration(this.conf, - YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + this.conf.addResource(this.configurationProvider + .getConfigurationInputStream( + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)); } // register the handlers for all AlwaysOn services using setupDispatcher(). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 8136c05..0ad6a1f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -164,10 +164,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + if (!(this.rmContext.getConfigurationProvider() + instanceof LocalConfigurationProvider)) { + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationInputStream( + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + } + refreshServiceAcls(conf,RMPolicyProvider.getInstance()); } this.server.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index eb4f814..2b7981e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -268,10 +268,10 @@ public Resource getClusterResources() { this.conf = new CapacitySchedulerConfiguration(configuration, true); } else { try { - this.conf = - new CapacitySchedulerConfiguration(rmContext - .getConfigurationProvider().getConfiguration(configuration, - YarnConfiguration.CS_CONFIGURATION_FILE), false); + configuration.addResource(rmContext.getConfigurationProvider() + .getConfigurationInputStream( + YarnConfiguration.CS_CONFIGURATION_FILE)); + this.conf = new CapacitySchedulerConfiguration(configuration, false); } catch (Exception e) { throw new IOException(e); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index ee008e9..63f6f6b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,6 +24,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.util.Set; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -34,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; @@ -366,6 +370,69 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } + @Test + public void testRefreshNodesWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshNodes(RefreshNodesRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testRefreshNodesWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch (Exception ex) { + // Expect exception here + } + + // upload default configurations + uploadDefaultConfiguration(); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts"); + if (excludeHostsFile.exists()) { + excludeHostsFile.delete(); + } + if (!excludeHostsFile.createNewFile()) { + Assert.fail("Can not create " + "excludeHosts"); + } + PrintWriter fileWriter = new PrintWriter(excludeHostsFile); + fileWriter.write("0.0.0.0:123"); + fileWriter.close(); + + uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath())); + + Configuration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath + + "/excludeHosts"); + uploadConfiguration(yarnConf, YarnConfiguration.YARN_SITE_XML_FILE); + + rm.adminService.refreshNodes(RefreshNodesRequest.newInstance()); + Set excludeHosts = + rm.getNodesListManager().getHostsReader().getExcludedHosts(); + Assert.assertTrue(excludeHosts.size() == 1); + Assert.assertTrue(excludeHosts.contains("0.0.0.0:123")); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null;