From 112c0b0f0c71941f21ab858ae53dd3152925de46 Mon Sep 17 00:00:00 2001 From: Ray Yang Date: Wed, 26 Jun 2019 10:24:52 -0700 Subject: [PATCH] YARN-9646 Use localhost when testing with the mini yarn cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When testing org.apache.hadoop.yarn.applications.distributedshell.TestDistributedShell#testDSShellWithoutDomain at home The following error happened: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.net.BindException: Problem binding to [ruyang-mn3.linkedin.biz:0] java.net.BindException: Can't assign requested address; For more details see:  http://wiki.apache.org/hadoop/BindException at org.apache.hadoop.yarn.server.MiniYARNCluster.startResourceManager(MiniYARNCluster.java:327) ... Caused by: java.net.BindException: Can't assign requested address at sun.nio.ch.Net.bind0(Native Method) It's because the test uses InetAddress.getLocalHost().getHostName(); to get the host name and tries to bind to it. The machine is issued at work. At home, the FQDN of the hostname is either not resolvable or the cached ip address is not reachable. Even on VPN, somehow the same error happened from time to time. The exact cause of that behavior on VPN is unknown yet. It makes WFH more difficult. The test will be simpler and more reliable if it simply uses "localhost". === The comment in org.apache.hadoop.yarn.applications.distributedshell.TestDistributedShell#checkHostname   * NetUtils.getHostname() returns a string in the form "hostname/ip".   * Sometimes the hostname we get is the FQDN and sometimes the short name. In   * addition, on machines with multiple network interfaces, it runs any one of   * the ips. suggests some complexities involved with this method of calling InetAddress.getLocalHost() === This test specifically checks for hostname patterns. Not sure why. This change will remove the check, which simplifies the test quite a bit. Testing: The failed test succeeds and all the distributed shell tests and mini cluster tests succeed --- .../TestDistributedShell.java | 64 +------------------ .../hadoop/yarn/server/MiniYARNCluster.java | 16 +---- 2 files changed, 3 insertions(+), 77 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index ba7bf7a560db..21bdd1277d05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -401,7 +401,6 @@ public void run() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(yarnCluster.getConfig())); yarnClient.start(); - String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; @@ -420,10 +419,9 @@ public void run() { continue; } errorMessage = - "Expected host name to start with '" + hostName + "', was '" - + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; - if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { + if (appReport.getRpcPort() == -1) { verified = true; } @@ -730,64 +728,6 @@ private void verifyEntityForTimelineV2(File entityFile, String expectedEvent, return argsList.toArray(new String[argsList.size()]); } - /* - * NetUtils.getHostname() returns a string in the form "hostname/ip". - * Sometimes the hostname we get is the FQDN and sometimes the short name. In - * addition, on machines with multiple network interfaces, it runs any one of - * the ips. The function below compares the returns values for - * NetUtils.getHostname() accounting for the conditions mentioned. - */ - private boolean checkHostname(String appHostname) throws Exception { - - String hostname = NetUtils.getHostname(); - if (hostname.equals(appHostname)) { - return true; - } - - Assert.assertTrue("Unknown format for hostname " + appHostname, - appHostname.contains("/")); - Assert.assertTrue("Unknown format for hostname " + hostname, - hostname.contains("/")); - - String[] appHostnameParts = appHostname.split("/"); - String[] hostnameParts = hostname.split("/"); - - return (compareFQDNs(appHostnameParts[0], hostnameParts[0]) && checkIPs( - hostnameParts[0], hostnameParts[1], appHostnameParts[1])); - } - - private boolean compareFQDNs(String appHostname, String hostname) - throws Exception { - if (appHostname.equals(hostname)) { - return true; - } - String appFQDN = InetAddress.getByName(appHostname).getCanonicalHostName(); - String localFQDN = InetAddress.getByName(hostname).getCanonicalHostName(); - return appFQDN.equals(localFQDN); - } - - private boolean checkIPs(String hostname, String localIP, String appIP) - throws Exception { - - if (localIP.equals(appIP)) { - return true; - } - boolean appIPCheck = false; - boolean localIPCheck = false; - InetAddress[] addresses = InetAddress.getAllByName(hostname); - for (InetAddress ia : addresses) { - if (ia.getHostAddress().equals(appIP)) { - appIPCheck = true; - continue; - } - if (ia.getHostAddress().equals(localIP)) { - localIPCheck = true; - } - } - return (appIPCheck && localIPCheck); - - } - protected String getSleepCommand(int sec) { // Windows doesn't have a sleep command, ping -n does the trick return Shell.WINDOWS ? "ping -n " + (sec + 1) + " 127.0.0.1 >nul" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 19c4eb4e1ab2..426bc068fdeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -470,21 +470,7 @@ public NodeManager getNodeManager(int i) { } public static String getHostname() { - try { - String hostname = InetAddress.getLocalHost().getHostName(); - // Create InetSocketAddress to see whether it is resolved or not. - // If not, just return "localhost". - InetSocketAddress addr = - NetUtils.createSocketAddrForHost(hostname, 1); - if (addr.isUnresolved()) { - return "localhost"; - } else { - return hostname; - } - } - catch (UnknownHostException ex) { - throw new RuntimeException(ex); - } + return "localhost"; } private class ResourceManagerWrapper extends AbstractService {