diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java index a85470155a4..0556b240b38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.util; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -86,6 +87,19 @@ public static Node resolve(Configuration conf, String hostName) { return coreResolve(hostName); } + /** + * Utility method for getting a list of hostname resolved to a list of node in the + * network topology. This method initializes the class with the + * right resolver implementation. + * @param conf + * @param hostNames + * @return nodes {@link Node} after resolving the hostnames + */ + public static List resolve(Configuration conf, List hostNames) { + init(conf); + return coreResolve(hostNames); + } + /** * Utility method for getting a hostname resolved to a node in the * network topology. This method doesn't initialize the class. @@ -100,18 +114,41 @@ public static Node resolve(String hostName) { return coreResolve(hostName); } + /** + * Utility method for getting a list of hostname resolved to a list of node in the + * network topology. This method doesn't initialize the class. + * Call {@link #init(Configuration)} explicitly. + * @param hostNames + * @return nodes {@link Node} after resolving the hostnames + */ + public static List resolve(List hostNames) { + if (!initCalled) { + throw new IllegalStateException("RackResolver class not yet initialized"); + } + return coreResolve(hostNames); + } + private static Node coreResolve(String hostName) { List tmpList = Collections.singletonList(hostName); - List rNameList = dnsToSwitchMapping.resolve(tmpList); - String rName = NetworkTopology.DEFAULT_RACK; - if (rNameList == null || rNameList.get(0) == null) { - LOG.debug("Could not resolve {}. Falling back to {}", hostName, - NetworkTopology.DEFAULT_RACK); + return coreResolve(tmpList).get(0); + } + + private static List coreResolve(List hostNames) { + List nodes = new ArrayList(hostNames.size()); + List rNameList = dnsToSwitchMapping.resolve(hostNames); + if (rNameList == null || rNameList.isEmpty()) { + for (String hostName : hostNames) { + nodes.add(new NodeBase(hostName, NetworkTopology.DEFAULT_RACK)); + } + LOG.info("Got an error when resolve hostNames. Falling back to " + + NetworkTopology.DEFAULT_RACK + " for all."); } else { - rName = rNameList.get(0); - LOG.debug("Resolved {} to {}", hostName, rName); + for (int i = 0; i < hostNames.size(); i++) { + LOG.debug("Resolved {} to {}", hostNames.get(i), rNameList.get(i)); + nodes.add(new NodeBase(hostNames.get(i), rNameList.get(i))); + } } - return new NodeBase(hostName, rName); + return nodes; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java index 70ca23c3a2e..bdf5418b964 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java @@ -18,9 +18,8 @@ package org.apache.hadoop.yarn.util; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.logging.Log; @@ -41,33 +40,27 @@ public static final class MyResolver implements DNSToSwitchMapping { - int numHost1 = 0; - public static String resolvedHost1 = "host1"; + public static List resolvedHosts = Arrays.asList("host1","host2"); @Override public List resolve(List hostList) { - // Only one host at a time - Assert.assertTrue("hostList size is " + hostList.size(), - hostList.size() <= 1); List returnList = new ArrayList(); if (hostList.isEmpty()) { return returnList; } - if (hostList.get(0).equals(invalidHost)) { - // Simulate condition where resolving host returns null - return null; + for (String host : hostList) { + if (host.equals(invalidHost)) { + // Simulate condition where resolving host returns null + return null; + } + LOG.info("Received resolve request for " + host); + if (host.startsWith("host")) { + returnList.add("/" + host.replace("host", "rack")); + } + // I should not be reached again as RackResolver is supposed to do + // caching. } - - LOG.info("Received resolve request for " - + hostList.get(0)); - if (hostList.get(0).equals("host1") - || hostList.get(0).equals(resolvedHost1)) { - numHost1++; - returnList.add("/rack1"); - } - // I should not be reached again as RackResolver is supposed to do - // caching. - Assert.assertTrue(numHost1 <= 1); + Assert.assertTrue(returnList.size() == hostList.size()); return returnList; } @@ -88,18 +81,15 @@ public void testCaching() { CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); - try { - InetAddress iaddr = InetAddress.getByName("host1"); - MyResolver.resolvedHost1 = iaddr.getHostAddress(); - } catch (UnknownHostException e) { - // Ignore if not found - } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); + List nodes = RackResolver.resolve(Arrays.asList("host1","host2")); + Assert.assertEquals("/rack1", nodes.get(0).getNetworkLocation()); + Assert.assertEquals("/rack2", nodes.get(1).getNetworkLocation()); } }