diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java index d557309..971c7dd 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java @@ -19,12 +19,15 @@ package org.apache.hadoop.util; import java.io.*; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Set; import java.util.HashSet; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; // Keeps track of which datanodes/tasktrackers are allowed to connect to the @@ -86,14 +89,59 @@ public synchronized void refresh() throws IOException { Set newIncludes = new HashSet(); readFileToSet("included", includesFile, newIncludes); // switch the new hosts that are to be included - includes = newIncludes; + includes.clear(); + for(String nodeAddress : newIncludes) { + if(validNodeAddress(nodeAddress.trim(), "included")) { + includes.add(nodeAddress); + } + } } if (!excludesFile.isEmpty()) { Set newExcludes = new HashSet(); readFileToSet("excluded", excludesFile, newExcludes); // switch the excluded hosts - excludes = newExcludes; + excludes.clear(); + for(String nodeAddress : newExcludes) { + if(validNodeAddress(nodeAddress.trim(), "excluded")) { + excludes.add(nodeAddress); + } + } + } + } + + protected boolean validNodeAddress(String nodeAddress, String fileName) { + final String prefix; + final int port; + String ipAddress = ""; + + int idx = nodeAddress.indexOf(':'); + if (-1 == idx) { + prefix = nodeAddress; + port = 0; + } else { + prefix = nodeAddress.substring(0, idx); + String portStr = nodeAddress.substring(idx + 1); + try { + port = Integer.valueOf(portStr); + } catch (NumberFormatException e) { + LOG.info("unable to parse port number for " + + "'" + nodeAddress + "'", e); + return false; + } + } + try { + // Let's see if we can resolve this prefix to an IP address. + // This may fail; one example is with a registered hostname + // which is not actually a real DNS name. + InetAddress addr = InetAddress.getByName(prefix); + ipAddress = addr.getHostAddress(); + } catch (UnknownHostException e) { + LOG.info("When reading " + fileName + ", could not look up " + + "IP address for " + prefix + ". We will assume this is a " + + "registration name.", e); + return false; } + return true; } public synchronized Set getHosts() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index e9a0436..98315c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.service.AbstractService; @@ -63,20 +64,21 @@ protected void serviceInit(Configuration conf) throws Exception { // Read the hosts/exclude files to restrict access to the RM try { - this.hostsReader = - new HostsFileReader( - conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), - conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH) - ); + this.hostsReader = + createHostsFileReader( + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH) + ); printConfiguredHosts(); } catch (IOException ioe) { LOG.warn("Failed to init hostsReader, disabling", ioe); try { - this.hostsReader = - new HostsFileReader(YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH, - YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); + this.hostsReader = + createHostsFileReader( + YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH, + YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); } catch (IOException ioe2) { // Should *never* happen this.hostsReader = null; @@ -177,4 +179,14 @@ public void handle(NodesListManagerEvent event) { LOG.error("Ignoring invalid eventtype " + event.getType()); } } + + protected HostsFileReader createHostsFileReader(String includes, + String excludes) throws IOException { + return new HostsFileReader(includes, excludes); + } + + @Private + public HostsFileReader getHostsFileReader() { + return this.hostsReader; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 6ca5307..924762f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -209,7 +209,7 @@ protected void serviceInit(Configuration conf) throws Exception { this.clientToAMSecretManager); // Register event handler for NodesListManager - this.nodesListManager = new NodesListManager(this.rmContext); + this.nodesListManager = createNodeListManager(this.rmContext); this.rmDispatcher.register(NodesListManagerEventType.class, this.nodesListManager); addService(nodesListManager); @@ -794,6 +794,10 @@ protected AdminService createAdminService( resourceTrackerService); } + protected NodesListManager createNodeListManager(RMContext rmContext) { + return new NodesListManager(rmContext); + } + @Private public ClientRMService getClientRMService() { return this.clientRM; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockHostsFileReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockHostsFileReader.java new file mode 100644 index 0000000..e386563 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockHostsFileReader.java @@ -0,0 +1,49 @@ +package org.apache.hadoop.yarn.server.resourcemanager; +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.util.HostsFileReader; + + +public class MockHostsFileReader extends HostsFileReader { + + private List acceptedHostNames = new ArrayList(); + + public MockHostsFileReader(String inFile, String exFile) throws IOException { + super(inFile, exFile); + } + + @Override + protected boolean validNodeAddress(String nodeAddress, String fileName) { + if (acceptedHostNames == null || acceptedHostNames.isEmpty() + || acceptedHostNames.contains(nodeAddress)) { + return true; + } + return false; + } + + public void setAcceptedHostNames(List acceptedHostNames) { + this.acceptedHostNames.clear(); + this.acceptedHostNames = acceptedHostNames; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 522debb..efa67c3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; @@ -379,6 +380,17 @@ protected void serviceStop() { }; } + @Override + protected NodesListManager createNodeListManager(RMContext rmContext) { + return new NodesListManager(rmContext) { + @Override + protected HostsFileReader createHostsFileReader(String includes, + String excludes) throws IOException { + return new MockHostsFileReader(includes, excludes); + } + }; + } + public NodesListManager getNodesListManager() { return this.nodesListManager; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 83a6af8..f656e70 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -29,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -79,6 +81,29 @@ public void testGetNextHeartBeatInterval() throws Exception { } + @Test(timeout = 5000) + public void testReadInvalidHostName() throws Exception { + writeToHostsFile("localhost", "host1", "host2", "InvalidHost"); + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile + .getAbsolutePath()); + rm = new MockRM(conf); + List acceptedHostsName = new ArrayList(); + acceptedHostsName.add("localhost"); + acceptedHostsName.add("host1"); + acceptedHostsName.add("host2"); + HostsFileReader mockHostsFileReader = + rm.getNodesListManager().getHostsFileReader(); + ((MockHostsFileReader) mockHostsFileReader) + .setAcceptedHostNames(acceptedHostsName); + rm.start(); + rm.getNodesListManager().refreshNodes(conf); + Assert.assertTrue(mockHostsFileReader.getHosts().containsAll( + acceptedHostsName)); + Assert.assertTrue(!mockHostsFileReader.getHosts().contains("InvalidHost")); + rm.stop(); + } + /** * Decommissioning using a pre-configured include hosts file */