.../org/apache/hadoop/fs/ServerSocketUtil.java | 60 ++++++++++++++++++++++ .../nodemanager/TestNodeManagerShutdown.java | 30 ++++++++--- .../containermanager/BaseContainerManagerTest.java | 6 +-- 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ServerSocketUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ServerSocketUtil.java new file mode 100644 index 0000000..5832ce9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ServerSocketUtil.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.Random; + +public class ServerSocketUtil { + /** + * Port scan & allocate is how most other apps find ports + * + * @return port + * @throws IOException + */ + public static int getPort() throws IOException { + Random rand = new Random(); + int port = -1; + int tries = 0; + while (port == -1) { + int tryPort = 49152 + rand.nextInt(65535 - 49152); + try (ServerSocket s = new ServerSocket(tryPort)) { + System.out.println("Using port " + tryPort); + port = tryPort; + } catch (IOException e) { + tries++; + if (tries >= 10) { + System.out.println("Port is already in use; giving up"); + throw new IOException(e); + } else { + System.out.println("Port is already in use; trying again"); + } + } + } + return port; + } + + /** + * @return 0 + */ + public int getFreePort() { + return 0; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index 2a887ba..a1a22de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -33,10 +33,10 @@ import java.util.Map; import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.ServerSocketUtil; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -195,7 +195,7 @@ public static void startContainer(NodeManager nm, ContainerId cId, recordFactory.newRecordInstance(ContainerLaunchContext.class); NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost") - .getCanonicalHostName(), 12345); + .getCanonicalHostName(), ServerSocketUtil.getPort()); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -215,7 +215,8 @@ public static void startContainer(NodeManager nm, ContainerId cId, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); final InetSocketAddress containerManagerBindAddress = - NetUtils.createSocketAddrForHost("127.0.0.1", 12345); + NetUtils.createSocketAddrForHost("127.0.0.1", + ServerSocketUtil.getPort()); UserGroupInformation currentUser = UserGroupInformation .createRemoteUser(cId.toString()); org.apache.hadoop.security.token.Token nmToken = @@ -231,8 +232,18 @@ public static void startContainer(NodeManager nm, ContainerId cId, public ContainerManagementProtocol run() { Configuration conf = new Configuration(); YarnRPC rpc = YarnRPC.create(conf); - InetSocketAddress containerManagerBindAddress = - NetUtils.createSocketAddrForHost("127.0.0.1", 12345); + InetSocketAddress containerManagerBindAddress = null; + try { + containerManagerBindAddress = + NetUtils.createSocketAddrForHost("127.0.0.1", + ServerSocketUtil.getPort()); + } catch (RuntimeException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf); } @@ -264,11 +275,14 @@ public static ContainerId createContainerId() { return containerId; } - private YarnConfiguration createNMConfig() { + private YarnConfiguration createNMConfig() throws RuntimeException, + IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB - conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345"); - conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346"); + conf.set(YarnConfiguration.NM_ADDRESS, + "127.0.0.1:" + ServerSocketUtil.getPort()); + conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:" + + ServerSocketUtil.getPort()); conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index a8e723d..c1f7ab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -28,12 +28,12 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.ServerSocketUtil; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -164,8 +164,8 @@ public void setup() throws IOException { remoteLogDir.mkdir(); LOG.info("Created localDir in " + localDir.getAbsolutePath()); LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath()); - - String bindAddress = "0.0.0.0:12345"; + String bindAddress = "0.0.0.0:" + ServerSocketUtil.getPort(); + // String bindAddress = "0.0.0.0:12345"; conf.set(YarnConfiguration.NM_ADDRESS, bindAddress); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());