.../server/nodemanager/TestNodeManagerReboot.java | 13 ++++++----- .../server/nodemanager/TestNodeManagerResync.java | 9 +++++--- .../nodemanager/TestNodeManagerShutdown.java | 25 +++++++++++++++------- .../server/nodemanager/TestNodeStatusUpdater.java | 13 ++++++----- .../containermanager/BaseContainerManagerTest.java | 4 ++-- .../TestContainerManagerRecovery.java | 7 ++++-- .../containermanager/container/TestContainer.java | 3 ++- .../monitor/TestContainersMonitor.java | 1 - .../TestNMContainerTokenSecretManager.java | 4 +++- .../security/TestNMTokenSecretManagerInNM.java | 4 +++- .../server/nodemanager/webapp/TestNMWebServer.java | 4 +++- .../ahs/TestRMApplicationHistoryWriter.java | 5 +++-- .../applicationsmanager/TestAMRMRPCResponseId.java | 4 +++- .../applicationsmanager/TestAMRestart.java | 21 +++++++++++------- .../TestRMAppLogAggregationStatus.java | 4 +++- 15 files changed, 79 insertions(+), 42 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index 41c16a9..4ad80f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -215,7 +216,7 @@ public Void run() throws YarnException, IOException { } - private void restartNM(int maxTries) { + private void restartNM(int maxTries) throws IOException { nm.stop(); nm = new MyNodeManager(); nm.start(); @@ -296,7 +297,7 @@ private ContainerId createContainerId() { private class MyNodeManager extends NodeManager { - public MyNodeManager() { + public MyNodeManager() throws IOException { super(); this.init(createNMConfig()); } @@ -315,11 +316,13 @@ protected DeletionService createDeletionService(ContainerExecutor exec) { return delService; } - private YarnConfiguration createNMConfig() { + private YarnConfiguration createNMConfig() throws IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB - conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345"); - conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346"); + conf.set(YarnConfiguration.NM_ADDRESS, + "127.0.0.1:" + ServerSocketUtil.getPort(49152, 10)); + conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:" + + ServerSocketUtil.getPort(49152, 10)); conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath()); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index 611e671..d32c4dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -327,11 +328,13 @@ protected void rebootNodeStatusUpdaterAndRegisterWithRM() { } } - private YarnConfiguration createNMConfig() { + private YarnConfiguration createNMConfig() throws IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB - conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345"); - conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346"); + conf.set(YarnConfiguration.NM_ADDRESS, + "127.0.0.1:" + ServerSocketUtil.getPort(49152, 10)); + conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:" + + ServerSocketUtil.getPort(49152, 10)); conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogsDir.getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index 2a887ba..e619002 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -33,12 +33,12 @@ import java.util.Map; import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; @@ -195,7 +195,7 @@ public static void startContainer(NodeManager nm, ContainerId cId, recordFactory.newRecordInstance(ContainerLaunchContext.class); NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost") - .getCanonicalHostName(), 12345); + .getCanonicalHostName(), ServerSocketUtil.getPort(49152, 10)); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -215,7 +215,9 @@ public static void startContainer(NodeManager nm, ContainerId cId, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); final InetSocketAddress containerManagerBindAddress = - NetUtils.createSocketAddrForHost("127.0.0.1", 12345); + NetUtils + .createSocketAddrForHost("127.0.0.1", + ServerSocketUtil.getPort(49152, 10)); UserGroupInformation currentUser = UserGroupInformation .createRemoteUser(cId.toString()); org.apache.hadoop.security.token.Token nmToken = @@ -231,8 +233,13 @@ public static void startContainer(NodeManager nm, ContainerId cId, public ContainerManagementProtocol run() { Configuration conf = new Configuration(); YarnRPC rpc = YarnRPC.create(conf); - InetSocketAddress containerManagerBindAddress = - NetUtils.createSocketAddrForHost("127.0.0.1", 12345); + InetSocketAddress containerManagerBindAddress; + try { + containerManagerBindAddress = NetUtils.createSocketAddrForHost( + "127.0.0.1", ServerSocketUtil.getPort(49152, 10)); + } catch (IOException e) { + throw new RuntimeException("Fail To Get the Port"); + } return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf); } @@ -264,11 +271,13 @@ public static ContainerId createContainerId() { return containerId; } - private YarnConfiguration createNMConfig() { + private YarnConfiguration createNMConfig() throws IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB - conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345"); - conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346"); + conf.set(YarnConfiguration.NM_ADDRESS, + "127.0.0.1:" + ServerSocketUtil.getPort(49152, 10)); + conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:" + + ServerSocketUtil.getPort(49152, 10)); conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index bc48adf..1c8fd47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -57,6 +57,7 @@ import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.service.Service.STATE; @@ -138,7 +139,7 @@ private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); @Before - public void setUp() { + public void setUp() throws IOException { nmLocalDir.mkdirs(); tmpDir.mkdirs(); logsDir.mkdirs(); @@ -1045,7 +1046,7 @@ public ContainerState getCurrentState() { } @Test - public void testNMRegistration() throws InterruptedException { + public void testNMRegistration() throws InterruptedException, IOException { nm = new NodeManager() { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, @@ -1465,7 +1466,8 @@ public void testNodeStatusUpdaterRetryAndNMShutdown() } @Test - public void testRMVersionLessThanMinimum() throws InterruptedException { + public void testRMVersionLessThanMinimum() throws InterruptedException, + IOException { final AtomicInteger numCleanups = new AtomicInteger(0); YarnConfiguration conf = createNMConfig(); conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0"); @@ -1673,7 +1675,7 @@ private void verifyNodeStartFailure(String errMessage) throws Exception { this.registeredNodes.size()); } - private YarnConfiguration createNMConfig() { + private YarnConfiguration createNMConfig() throws IOException { YarnConfiguration conf = new YarnConfiguration(); String localhostAddress = null; try { @@ -1682,7 +1684,8 @@ private YarnConfiguration createNMConfig() { Assert.fail("Unable to get localhost address: " + e.getMessage()); } conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB - conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345"); + conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":" + + ServerSocketUtil.getPort(49152, 10)); conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346"); conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index a8e723d..8c67cf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -28,13 +28,13 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.util.NodeHealthScriptRunner; @@ -165,7 +165,7 @@ public void setup() throws IOException { LOG.info("Created localDir in " + localDir.getAbsolutePath()); LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath()); - String bindAddress = "0.0.0.0:12345"; + String bindAddress = "0.0.0.0:" + ServerSocketUtil.getPort(49152, 10); conf.set(YarnConfiguration.NM_ADDRESS, bindAddress); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index 781950e..30df498 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; @@ -88,7 +89,8 @@ public void testApplicationRecovery() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true); - conf.set(YarnConfiguration.NM_ADDRESS, "localhost:1234"); + conf.set(YarnConfiguration.NM_ADDRESS, + "localhost:" + ServerSocketUtil.getPort(49152, 10)); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "yarn_admin_user"); NMStateStoreService stateStore = new NMMemoryStateStoreService(); @@ -260,7 +262,8 @@ public void testContainerCleanupOnShutdown() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, false); conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, false); - conf.set(YarnConfiguration.NM_ADDRESS, "localhost:1234"); + conf.set(YarnConfiguration.NM_ADDRESS, + "localhost:" + ServerSocketUtil.getPort(49152, 10)); Context context = createContext(conf, new NMNullStateStoreService()); ContainerManagerImpl cm = spy(createContainerManager(context)); cm.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 2834e30..d30e1c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -50,6 +50,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -776,7 +777,7 @@ public boolean matches(Object o) { Resource resource = BuilderUtils.newResource(1024, 1); when(mockContainer.getResource()).thenReturn(resource); String host = "127.0.0.1"; - int port = 1234; + int port = ServerSocketUtil.getPort(49152, 10); long currentTime = System.currentTimeMillis(); ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 909a962..6e37b95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -208,7 +208,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, ApplicationId appId = ApplicationId.newInstance(0, 0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId cId = ContainerId.newContainerId(appAttemptId, 0); - int port = 12345; URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java index f2a46ad..39bf1d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java @@ -27,6 +27,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -46,7 +47,8 @@ public void testRecovery() throws IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); - final NodeId nodeId = NodeId.newInstance("somehost", 1234); + final NodeId nodeId = NodeId.newInstance("somehost", + ServerSocketUtil.getPort(49152, 10)); final ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1); final ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2); ContainerTokenKeyGeneratorForTest keygen = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java index 5c1f3a1..2af7db0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java @@ -27,6 +27,7 @@ import java.io.IOException; import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -46,7 +47,8 @@ public void testRecovery() throws IOException { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); - final NodeId nodeId = NodeId.newInstance("somehost", 1234); + final NodeId nodeId = NodeId.newInstance("somehost", + ServerSocketUtil.getPort(49152, 10)); final ApplicationAttemptId attempt1 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); final ApplicationAttemptId attempt2 = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index e1845c7..84d1805 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.util.NodeHealthScriptRunner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -212,7 +213,8 @@ public boolean isPmemCheckEnabled() { recordFactory.newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); Token containerToken = - BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, + BuilderUtils.newContainerToken(containerId, + "127.0.0.1", ServerSocketUtil.getPort(49152, 10), user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); Container container = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index f827bf4..73f3c4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -28,8 +28,8 @@ import java.util.Random; import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -443,7 +443,8 @@ public void containerFinished(RMContainer container) { private void testRMWritingMassiveHistory(MockRM rm) throws Exception { rm.start(); - MockNM nm = rm.registerNode("127.0.0.1:1234", 1024 * 10100); + MockNM nm = rm.registerNode( + "127.0.0.1:" + ServerSocketUtil.getPort(49152, 10), 1024 * 10100); RMApp app = rm.submitApp(1024); nm.nodeHeartbeat(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java index d609d7a..4e264de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java @@ -20,6 +20,7 @@ import java.security.PrivilegedExceptionAction; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -75,7 +76,8 @@ public AllocateResponse run() throws Exception { @Test public void testARRMResponseId() throws Exception { - MockNM nm1 = rm.registerNode("h1:1234", 5000); + MockNM nm1 = rm.registerNode("h1:" + ServerSocketUtil.getPort(49152, 10), + 5000); RMApp app = rm.submitApp(2000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index d579595..9cd0647 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -74,10 +75,12 @@ public void testAMRestartWithExistingContainers() throws Exception { new HashMap(), false, "default", -1, null, "MAPREDUCE", false, true); MockNM nm1 = - new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:" + ServerSocketUtil.getPort(49152, 10), + 10240, rm1.getResourceTrackerService()); nm1.registerNode(); MockNM nm2 = - new MockNM("127.0.0.1:2351", 4089, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:" + ServerSocketUtil.getPort(49152, 10), + 4089, rm1.getResourceTrackerService()); nm2.registerNode(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -266,10 +269,12 @@ public void testNMTokensRebindOnAMRestart() throws Exception { new HashMap(), false, "default", -1, null, "MAPREDUCE", false, true); MockNM nm1 = - new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:" + ServerSocketUtil.getPort(49152, 10), + 8000, rm1.getResourceTrackerService()); nm1.registerNode(); MockNM nm2 = - new MockNM("127.1.1.1:4321", 8000, rm1.getResourceTrackerService()); + new MockNM("127.1.1.1:" + ServerSocketUtil.getPort(49152, 10), + 8000, rm1.getResourceTrackerService()); nm2.registerNode(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); @@ -371,7 +376,7 @@ public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception { MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = - new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:"+ServerSocketUtil.getPort(49152, 10), 8000, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -481,7 +486,7 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = - new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:"+ServerSocketUtil.getPort(49152, 10), 8000, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); @@ -548,7 +553,7 @@ public void testRMRestartOrFailoverNotCountedForAMFailures() MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = - new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:"+ServerSocketUtil.getPort(49152, 10), 8000, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); // AM should be restarted even though max-am-attempt is 1. @@ -609,7 +614,7 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception { MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = - new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + new MockNM("127.0.0.1:"+ServerSocketUtil.getPort(49152, 10), 8000, rm1.getResourceTrackerService()); nm1.registerNode(); // set window size to a larger number : 60s diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java index 9af4290..3e0674b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java @@ -28,6 +28,7 @@ import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -129,7 +130,8 @@ public void testLogAggregationStatus() throws Exception { rmApp.handle(new RMAppEvent(this.appId, RMAppEventType.APP_ACCEPTED)); // This application will be running on two nodes - NodeId nodeId1 = NodeId.newInstance("localhost", 1234); + NodeId nodeId1 = NodeId.newInstance("localhost", + ServerSocketUtil.getPort(49152, 10)); Resource capability = Resource.newInstance(4096, 4); RMNodeImpl node1 = new RMNodeImpl(nodeId1, rmContext, null, 0, 0, null, capability, null);