diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java index d557309..18888cb 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; // Keeps track of which datanodes/tasktrackers are allowed to connect to the @@ -48,6 +49,12 @@ public HostsFileReader(String inFile, refresh(); } + @Private + public HostsFileReader(Set includesFile, Set excludesFile) { + this.includes = includesFile; + this.excludes = excludesFile; + } + public static void readFileToSet(String type, String filename, Set set) throws IOException { File file = new File(filename); @@ -96,6 +103,14 @@ public synchronized void refresh() throws IOException { } } + @Private + public synchronized void refresh(Set includesHosts, + Set excludesHosts) throws IOException { + LOG.info("Refreshing hosts (include/exclude) list"); + this.includes = includesHosts; + this.excludes = excludesHosts; + } + public synchronized Set getHosts() { return includes; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java index b31573d..1ce77ba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.conf; import java.io.IOException; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -53,6 +55,15 @@ public abstract Configuration getConfiguration(Configuration bootstrapConf, String name) throws YarnException, IOException; /** + * Get exclude or include hosts list + * @param name The host file name + * @return hostList + * @throws YarnException + * @throws IOException + */ + public abstract Set getHostsList(String name) throws YarnException, + IOException; + /** * Derived classes initialize themselves using this method. */ public abstract void initInternal(Configuration bootstrapConf) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index 390aace..e00e76e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -18,7 +18,14 @@ package org.apache.hadoop.yarn; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -53,6 +60,45 @@ public synchronized Configuration getConfiguration(Configuration bootstrapConf, } @Override + public synchronized Set getHostsList(String name) throws YarnException, + IOException { + Set set = new HashSet(); + if (name.isEmpty()) { + return set; + } + Path filePath = new Path(name); + if (!fs.exists(filePath)) { + throw new IOException("Can not find the file:" + name); + } + InputStream inputStream = fs.open(filePath); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = reader.readLine()) != null) { + String[] nodes = line.split("[ \t\n\f\r]+"); + if (nodes != null) { + for (int i = 0; i < nodes.length; i++) { + if (nodes[i].trim().startsWith("#")) { + // Everything from now on is a comment + break; + } + if (!nodes[i].isEmpty()) { + set.add(nodes[i]); + } + } + } + } + } finally { + if (reader != null) { + IOUtils.closeQuietly(reader); + } + IOUtils.closeQuietly(inputStream); + } + return set; + } + + @Override public synchronized void initInternal(Configuration bootstrapConf) throws Exception { configDir = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index 3e69960..51aa073 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -36,6 +39,12 @@ public Configuration getConfiguration(Configuration bootstrapConf, } @Override + public Set getHostsList(String name) throws YarnException, + IOException { + return new HashSet(); + } + + @Override public void initInternal(Configuration bootstrapConf) throws Exception { // Do nothing } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index da479b4..e5a2d81 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -331,23 +331,26 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) @Override public RefreshNodesResponse refreshNodes(RefreshNodesRequest request) throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshNodes"); + String argName = "refreshNodes"; + UserGroupInformation user = checkAcls(argName); if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh nodes."); throwStandbyException(); } try { - rmContext.getNodesListManager().refreshNodes(new YarnConfiguration()); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshNodes", + rmContext.getNodesListManager().refreshNodes( + getConfiguration(new YarnConfiguration(), + YarnConfiguration.YARN_SITE_XML_FILE)); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshNodesResponse.class); } catch (IOException ioe) { LOG.info("Exception refreshing nodes ", ioe); - RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "Exception refreshing nodes"); throw RPCUtil.getRemoteException(ioe); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index 4249980..480a0df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -31,13 +31,17 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import com.google.common.annotations.VisibleForTesting; + @SuppressWarnings("unchecked") public class NodesListManager extends AbstractService implements EventHandler { @@ -63,13 +67,27 @@ protected void serviceInit(Configuration conf) throws Exception { // Read the hosts/exclude files to restrict access to the RM try { - this.hostsReader = - new HostsFileReader( - conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), - conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH) - ); + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { + this.hostsReader = + new HostsFileReader( + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH) + ); + } else { + Set includesHost = + this.rmContext.getConfigurationProvider().getHostsList( + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH)); + + Set excludesHost = + this.rmContext.getConfigurationProvider().getHostsList( + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); + this.hostsReader = new HostsFileReader(includesHost, excludesHost); + } printConfiguredHosts(); } catch (IOException ioe) { LOG.warn("Failed to init hostsReader, disabling", ioe); @@ -103,17 +121,33 @@ private void printConfiguredHosts() { } } - public void refreshNodes(Configuration yarnConf) throws IOException { + public void refreshNodes(Configuration yarnConf) throws IOException, + YarnException { synchronized (hostsReader) { if (null == yarnConf) { yarnConf = new YarnConfiguration(); } - hostsReader.updateFileNames(yarnConf.get( - YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), yarnConf.get( - YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); - hostsReader.refresh(); + if (this.rmContext.getConfigurationProvider() instanceof + LocalConfigurationProvider) { + hostsReader.updateFileNames(yarnConf.get( + YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), + yarnConf.get( + YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); + hostsReader.refresh(); + } else { + Set includesHost = + this.rmContext.getConfigurationProvider().getHostsList( + yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH)); + + Set excludesHost = + this.rmContext.getConfigurationProvider().getHostsList( + yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); + hostsReader.refresh(includesHost, excludesHost); + } printConfiguredHosts(); } } @@ -174,4 +208,9 @@ public void handle(NodesListManagerEvent event) { LOG.error("Ignoring invalid eventtype " + event.getType()); } } + + @VisibleForTesting + public HostsFileReader getHostsReader() { + return this.hostsReader; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index ee008e9..b304075 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,6 +24,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.util.Set; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -34,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; @@ -366,6 +370,68 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } + @Test + public void testRefreshNodesWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshNodes(RefreshNodesRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testRefreshNodesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } + + //upload default configurations + uploadDefaultConfiguration(); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); + } + + final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts"); + if (excludeHostsFile.exists()) { + excludeHostsFile.delete(); + } + if (!excludeHostsFile.createNewFile()) { + Assert.fail("Can not create " + "excludeHosts"); + } + PrintWriter fileWriter = new PrintWriter(excludeHostsFile); + fileWriter.write("0.0.0.0:123"); + fileWriter.close(); + + uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath())); + + Configuration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath + + "/excludeHosts"); + uploadConfiguration(yarnConf, YarnConfiguration.YARN_SITE_XML_FILE); + + rm.adminService.refreshNodes(RefreshNodesRequest.newInstance()); + Set excludeHosts = + rm.getNodesListManager().getHostsReader().getExcludedHosts(); + Assert.assertTrue(excludeHosts.size() == 1); + Assert.assertTrue(excludeHosts.contains("0.0.0.0:123")); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null;