commit c96a4fe2d79051ff88be2a96aa0b5e9e25a544f4 Author: Vinod Kumar Vavilapalli Date: Fri May 24 15:28:34 2013 -0700 The final startContainer patch. diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index b483486..11c8343 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -154,7 +154,8 @@ public synchronized void launch(ContainerRemoteLaunchEvent event) { StartContainerRequest startRequest = Records .newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(event.getAllocatedContainer()); + startRequest.setContainerToken(event.getAllocatedContainer() + .getContainerToken()); StartContainerResponse response = proxy.startContainer(startRequest); ByteBuffer portInfo = diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 02c8e6b..786a9e3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; @@ -376,13 +377,12 @@ public GetContainerStatusResponse getContainerStatus( public StartContainerResponse startContainer(StartContainerRequest request) throws IOException { + ContainerTokenIdentifier containerTokenIdentifier = + BuilderUtils.newContainerTokenIdentifier(request.getContainerToken()); + // Validate that the container is what RM is giving. - Assert.assertEquals(MRApp.NM_HOST, request.getContainer().getNodeId() - .getHost()); - Assert.assertEquals(MRApp.NM_PORT, request.getContainer().getNodeId() - .getPort()); - Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_HTTP_PORT, request - .getContainer().getNodeHttpAddress()); + Assert.assertEquals(MRApp.NM_HOST, + containerTokenIdentifier.getNmHostAddress()); StartContainerResponse response = recordFactory .newRecordInstance(StartContainerResponse.class); @@ -395,7 +395,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) throw new UndeclaredThrowableException(e); } status.setState(ContainerState.RUNNING); - status.setContainerId(request.getContainer().getId()); + status.setContainerId(containerTokenIdentifier.getContainerID()); status.setExitStatus(0); return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java index 78862b9..ca161d5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java @@ -21,8 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ContainerManager; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerToken; /** *

The request sent by the ApplicationMaster to the @@ -62,9 +62,9 @@ @Public @Stable - public Container getContainer(); + public ContainerToken getContainerToken(); @Public @Stable - public void setContainer(Container container); + public void setContainerToken(ContainerToken container); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java index d4190e0..5073f0f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java @@ -19,14 +19,14 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerTokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProtoOrBuilder; @@ -39,7 +39,7 @@ private ContainerLaunchContext containerLaunchContext = null; - private Container container = null; + private ContainerToken containerToken = null; public StartContainerRequestPBImpl() { builder = StartContainerRequestProto.newBuilder(); @@ -61,8 +61,8 @@ private void mergeLocalToBuilder() { if (this.containerLaunchContext != null) { builder.setContainerLaunchContext(convertToProtoFormat(this.containerLaunchContext)); } - if(this.container != null) { - builder.setContainer(convertToProtoFormat(this.container)); + if(this.containerToken != null) { + builder.setContainerToken(convertToProtoFormat(this.containerToken)); } } @@ -104,25 +104,25 @@ public void setContainerLaunchContext(ContainerLaunchContext containerLaunchCont } @Override - public Container getContainer() { + public ContainerToken getContainerToken() { StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.container != null) { - return this.container; + if (this.containerToken != null) { + return this.containerToken; } - if (!p.hasContainer()) { + if (!p.hasContainerToken()) { return null; } - this.container = convertFromProtoFormat(p.getContainer()); - return this.container; + this.containerToken = convertFromProtoFormat(p.getContainerToken()); + return this.containerToken; } @Override - public void setContainer(Container container) { + public void setContainerToken(ContainerToken containerToken) { maybeInitBuilder(); - if(container == null) { - builder.clearContainer(); + if(containerToken == null) { + builder.clearContainerToken(); } - this.container = container; + this.containerToken = containerToken; } private ContainerLaunchContextPBImpl convertFromProtoFormat(ContainerLaunchContextProto p) { @@ -135,11 +135,11 @@ private ContainerLaunchContextProto convertToProtoFormat(ContainerLaunchContext - private ContainerPBImpl convertFromProtoFormat(ContainerProto containerProto) { - return new ContainerPBImpl(containerProto); + private ContainerTokenPBImpl convertFromProtoFormat(TokenProto containerProto) { + return new ContainerTokenPBImpl(containerProto); } - private ContainerProto convertToProtoFormat(Container container) { - return ((ContainerPBImpl)container).getProto(); + private TokenProto convertToProtoFormat(ContainerToken container) { + return ((ContainerTokenPBImpl)container).getProto(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 6ac0274..2b2c86b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -172,7 +172,7 @@ message GetQueueUserAclsInfoResponseProto { message StartContainerRequestProto { optional ContainerLaunchContextProto container_launch_context = 1; - optional ContainerProto container = 2; + optional hadoop.common.TokenProto container_token = 2; } message StartContainerResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 9f1437e..2f9f359 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -763,7 +763,7 @@ public void run() { StartContainerRequest startReq = Records .newRecord(StartContainerRequest.class); startReq.setContainerLaunchContext(ctx); - startReq.setContainer(container); + startReq.setContainerToken(container.getContainerToken()); try { cm.startContainer(startReq); } catch (YarnRemoteException e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java index 85c313c..2bf9c93 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java @@ -212,7 +212,7 @@ public synchronized void stop() { try { StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); - startRequest.setContainer(container); + startRequest.setContainerToken(container.getContainerToken()); startRequest.setContainerLaunchContext(containerLaunchContext); startResponse = containerManager.startContainer(startRequest); if (LOG.isDebugEnabled()) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index d3e9bce..27507e8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -37,20 +37,18 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; +import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; @@ -89,26 +87,10 @@ private void testRPCTimeout(String rpcClass) throws Exception { server.getListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - ContainerId containerId = recordFactory - .newRecordInstance(ContainerId.class); - ApplicationId applicationId = recordFactory - .newRecordInstance(ApplicationId.class); - ApplicationAttemptId applicationAttemptId = recordFactory - .newRecordInstance(ApplicationAttemptId.class); - applicationId.setClusterTimestamp(0); - applicationId.setId(0); - applicationAttemptId.setApplicationId(applicationId); - applicationAttemptId.setAttemptId(0); - containerId.setApplicationAttemptId(applicationAttemptId); - containerId.setId(100); - Container container = - BuilderUtils.newContainer(containerId, null, null, recordFactory - .newRecordInstance(Resource.class), null, null); - StartContainerRequest scRequest = recordFactory .newRecordInstance(StartContainerRequest.class); scRequest.setContainerLaunchContext(containerLaunchContext); - scRequest.setContainer(container); + scRequest.setContainerToken(null); try { proxy.startContainer(scRequest); } catch (Exception e) { @@ -151,8 +133,17 @@ public StartContainerResponse startContainer(StartContainerRequest request) LOG.error(e); throw new YarnRemoteException(e); } + + ContainerToken containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } status.setState(ContainerState.RUNNING); - status.setContainerId(request.getContainer().getId()); + status.setContainerId(tokenId.getContainerID()); status.setExitStatus(0); return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 61ca6ea..a2eaff8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn; +import java.io.IOException; import java.net.InetSocketAddress; import junit.framework.Assert; @@ -39,18 +40,19 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; +import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; @@ -123,21 +125,16 @@ private void test(String rpcClass) throws Exception { applicationAttemptId.setAttemptId(0); containerId.setApplicationAttemptId(applicationAttemptId); containerId.setId(100); - Container mockContainer = - BuilderUtils.newContainer(containerId, null, null, recordFactory - .newRecordInstance(Resource.class), null, null); -// containerLaunchContext.env = new HashMap(); -// containerLaunchContext.command = new ArrayList(); StartContainerRequest scRequest = recordFactory.newRecordInstance(StartContainerRequest.class); scRequest.setContainerLaunchContext(containerLaunchContext); - scRequest.setContainer(mockContainer); + scRequest.setContainerToken(null); proxy.startContainer(scRequest); GetContainerStatusRequest gcsRequest = recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(mockContainer.getId()); + gcsRequest.setContainerId(containerId); GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest); ContainerStatus status = response.getStatus(); @@ -145,7 +142,7 @@ private void test(String rpcClass) throws Exception { boolean exception = false; try { StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(mockContainer.getId()); + stopRequest.setContainerId(containerId); proxy.stopContainer(stopRequest); } catch (YarnRemoteException e) { exception = true; @@ -179,11 +176,19 @@ public GetContainerStatusResponse getContainerStatus( @Override public StartContainerResponse startContainer(StartContainerRequest request) throws YarnRemoteException { + ContainerToken containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); status = recordFactory.newRecordInstance(ContainerStatus.class); status.setState(ContainerState.RUNNING); - status.setContainerId(request.getContainer().getId()); + status.setContainerId(tokenId.getContainerID()); status.setExitStatus(0); return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index 54fbdb5..80fcc49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -42,6 +42,13 @@ */ NodeId getNodeId(); + /** + * Return the node http-address. Usable only after the Webserver is started. + * + * @return the http-port + */ + int getHttpPort(); + ConcurrentMap getApplications(); ConcurrentMap getContainers(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 89032f2..5b534ba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -113,7 +113,7 @@ public int launchContainer(Container container, List localDirs, List logDirs) throws IOException { FsPermission dirPerm = new FsPermission(APPDIR_PERM); - ContainerId containerId = container.getContainer().getId(); + ContainerId containerId = container.getContainerId(); // create container dirs on all disks String containerIdStr = ConverterUtils.toString(containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 0c6b502..4e6cdcb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -216,11 +216,11 @@ public int launchContainer(Container container, String user, String appId, Path containerWorkDir, List localDirs, List logDirs) throws IOException { - ContainerId containerId = container.getContainer().getId(); + ContainerId containerId = container.getContainerId(); String containerIdStr = ConverterUtils.toString(containerId); resourcesHandler.preExecute(containerId, - container.getContainer().getResource()); + container.getResource()); String resourcesOptions = resourcesHandler.getResourcesOption( containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 85f5784..2ae578b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; @@ -56,7 +55,6 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.CompositeService; -import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.util.Records; import com.google.common.annotations.VisibleForTesting; @@ -173,9 +171,10 @@ public void init(Configuration conf) { addService(containerManager); ((NMContext) context).setContainerManager(containerManager); - Service webServer = createWebServer(context, containerManager + WebServer webServer = createWebServer(context, containerManager .getContainersMonitor(), this.aclsManager, dirsHandler); addService(webServer); + ((NMContext) context).setWebServer(webServer); dispatcher.register(ContainerManagerEventType.class, containerManager); dispatcher.register(NodeManagerEventType.class, this); @@ -297,6 +296,7 @@ protected void cleanupContainers(NodeManagerEventType eventType) { private final NMContainerTokenSecretManager containerTokenSecretManager; private ContainerManager containerManager; + private WebServer webServer; private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider .getRecordFactory(null).newRecordInstance(NodeHealthStatus.class); @@ -316,6 +316,11 @@ public NodeId getNodeId() { } @Override + public int getHttpPort() { + return this.webServer.getPort(); + } + + @Override public ConcurrentMap getApplications() { return this.applications; } @@ -342,6 +347,10 @@ public ContainerManager getContainerManager() { public void setContainerManager(ContainerManager containerManager) { this.containerManager = containerManager; } + + public void setWebServer(WebServer webServer) { + this.webServer = webServer; + } } @@ -352,12 +361,6 @@ public NodeHealthCheckerService getNodeHealthChecker() { return nodeHealthChecker; } - private void reboot() { - LOG.info("Rebooting the node manager."); - NodeManager nodeManager = createNewNodeManager(); - nodeManager.initAndStartNodeManager(this.getConfig(), true); - } - private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) { try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index b33e7b3..53c01d9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -155,13 +155,8 @@ public void start() { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); - - InetSocketAddress httpBindAddress = getConfig().getSocketAddr( - YarnConfiguration.NM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_NM_WEBAPP_PORT); + this.httpPort = this.context.getHttpPort(); try { - this.httpPort = httpBindAddress.getPort(); // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. registerWithRM(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 6ffaa99..89ccb87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -52,7 +52,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -272,14 +272,13 @@ public void stop() { } // Get the remoteUGI corresponding to the api call. - private UserGroupInformation getRemoteUgi(String containerIDStr) + private UserGroupInformation getRemoteUgi() throws YarnRemoteException { UserGroupInformation remoteUgi; try { remoteUgi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { - String msg = "Cannot obtain the user-name for containerId: " - + containerIDStr + ". Got exception: " + String msg = "Cannot obtain the user-name. Got exception: " + StringUtils.stringifyException(e); LOG.warn(msg); throw RPCUtil.getRemoteException(msg); @@ -307,7 +306,7 @@ private ContainerTokenIdentifier selectContainerTokenIdentifier( @VisibleForTesting protected ContainerTokenIdentifier getContainerTokenIdentifier( UserGroupInformation remoteUgi, - org.apache.hadoop.yarn.api.records.Container container) + ContainerTokenIdentifier containerTokenIdentifier) throws YarnRemoteException { if (UserGroupInformation.isSecurityEnabled()) { if (LOG.isDebugEnabled()) { @@ -317,12 +316,7 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier( // Get the tokenId from the remote user ugi return selectContainerTokenIdentifier(remoteUgi); } else { - try { - return BuilderUtils.newContainerTokenIdentifier(container - .getContainerToken()); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); - } + return containerTokenIdentifier; } } @@ -341,7 +335,6 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier( @VisibleForTesting protected void authorizeRequest(String containerIDStr, ContainerLaunchContext launchContext, - org.apache.hadoop.yarn.api.records.Container container, UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId) throws YarnRemoteException { @@ -380,13 +373,6 @@ protected void authorizeRequest(String containerIDStr, + System.currentTimeMillis() + " found " + tokenId.getExpiryTimeStamp()); } - - Resource resource = tokenId.getResource(); - if (resource == null || !resource.equals(container.getResource())) { - unauthorized = true; - messageBuilder.append("\nExpected resource " + resource - + " but found " + container.getResource()); - } } } @@ -412,16 +398,23 @@ public StartContainerResponse startContainer(StartContainerRequest request) } ContainerLaunchContext launchContext = request.getContainerLaunchContext(); - org.apache.hadoop.yarn.api.records.Container lauchContainer = - request.getContainer(); - ContainerId containerID = lauchContainer.getId(); - String containerIDStr = containerID.toString(); + ContainerToken token = request.getContainerToken(); - UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + ContainerTokenIdentifier tokenIdentifier = null; + try { + tokenIdentifier = BuilderUtils.newContainerTokenIdentifier(token); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + + UserGroupInformation remoteUgi = getRemoteUgi(); ContainerTokenIdentifier tokenId = - getContainerTokenIdentifier(remoteUgi, lauchContainer); - authorizeRequest(containerIDStr, launchContext, lauchContainer, remoteUgi, - tokenId); + getContainerTokenIdentifier(remoteUgi, tokenIdentifier); + + ContainerId containerID = tokenId.getContainerID(); + String containerIDStr = containerID.toString(); + + authorizeRequest(containerIDStr, launchContext, remoteUgi, tokenId); // Is the container coming from unknown RM if (tokenId.getRMIdentifer() != nodeStatusUpdater @@ -458,8 +451,9 @@ public StartContainerResponse startContainer(StartContainerRequest request) // //////////// End of parsing credentials String user = tokenId.getApplicationSubmitter(); - Container container = new ContainerImpl(getConfig(), this.dispatcher, - launchContext, lauchContainer, credentials, metrics, tokenId); + Container container = + new ContainerImpl(getConfig(), this.dispatcher, launchContext, + credentials, metrics, tokenId); ApplicationId applicationID = containerID.getApplicationAttemptId().getApplicationId(); if (context.getContainers().putIfAbsent(containerID, container) != null) { @@ -501,7 +495,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) // TODO launchedContainer misplaced -> doesn't necessarily mean a container // launch. A finished Application will not launch containers. metrics.launchedContainer(); - metrics.allocateContainer(lauchContainer.getResource()); + metrics.allocateContainer(tokenId.getResource()); return response; } @@ -518,7 +512,7 @@ public StopContainerResponse stopContainer(StopContainerRequest request) // TODO: Only the container's owner can kill containers today. - UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + UserGroupInformation remoteUgi = getRemoteUgi(); Container container = this.context.getContainers().get(containerID); StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); @@ -532,8 +526,8 @@ public StopContainerResponse stopContainer(StopContainerRequest request) containerID); return response; // Return immediately. } - authorizeRequest(containerIDStr, null, null, remoteUgi, - getContainerTokenIdentifier(remoteUgi, container.getContainer())); + authorizeRequest(containerIDStr, null, remoteUgi, + getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier())); dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, @@ -561,15 +555,15 @@ public GetContainerStatusResponse getContainerStatus( // TODO: Only the container's owner can get containers' status today. - UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); + UserGroupInformation remoteUgi = getRemoteUgi(); LOG.info("Getting container-status for " + containerIDStr); Container container = this.context.getContainers().get(containerID); if (container == null) { throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } - authorizeRequest(containerIDStr, null, null, remoteUgi, - getContainerTokenIdentifier(remoteUgi, container.getContainer())); + authorizeRequest(containerIDStr, null, remoteUgi, + getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier())); ContainerStatus containerStatus = container.cloneAndGetContainerStatus(); LOG.info("Returning " + containerStatus); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java index b74be63..ef9e5a4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java @@ -36,7 +36,7 @@ final Container container; public ApplicationContainerInitEvent(Container container) { - super(container.getContainer().getId().getApplicationAttemptId() + super(container.getContainerId().getApplicationAttemptId() .getApplicationId(), ApplicationEventType.INIT_CONTAINER); this.container = container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index dc065a5..7a2ccb8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -273,14 +273,14 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { ApplicationContainerInitEvent initEvent = (ApplicationContainerInitEvent) event; Container container = initEvent.getContainer(); - app.containers.put(container.getContainer().getId(), container); - LOG.info("Adding " + container.getContainer().getId() + app.containers.put(container.getContainerId(), container); + LOG.info("Adding " + container.getContainerId() + " to application " + app.toString()); switch (app.getApplicationState()) { case RUNNING: app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainer().getId())); + container.getContainerId())); break; case INITING: case NEW: @@ -301,7 +301,7 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { // Start all the containers waiting for ApplicationInit for (Container container : app.containers.values()) { app.dispatcher.getEventHandler().handle(new ContainerInitEvent( - container.getContainer().getId())); + container.getContainerId())); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index c975101..e69e61a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -23,13 +23,20 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; public interface Container extends EventHandler { - org.apache.hadoop.yarn.api.records.Container getContainer(); + ContainerId getContainerId(); + + Resource getResource(); + + ContainerTokenIdentifier getContainerTokenIdentifier(); String getUser(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 8a95043..012b764 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -37,11 +37,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -76,7 +78,9 @@ private final Credentials credentials; private final NodeManagerMetrics metrics; private final ContainerLaunchContext launchContext; - private final org.apache.hadoop.yarn.api.records.Container container; + private final ContainerTokenIdentifier containerTokenIdentifier; + private final ContainerId containerId; + private final Resource resource; private final String user; private int exitCode = ContainerExitStatus.INVALID; private final StringBuilder diagnostics; @@ -97,18 +101,19 @@ new ArrayList(); public ContainerImpl(Configuration conf, Dispatcher dispatcher, - ContainerLaunchContext launchContext, - org.apache.hadoop.yarn.api.records.Container container, - Credentials creds, NodeManagerMetrics metrics, - ContainerTokenIdentifier identifier) throws IOException { + ContainerLaunchContext launchContext, Credentials creds, + NodeManagerMetrics metrics, + ContainerTokenIdentifier containerTokenIdentifier) throws IOException { this.daemonConf = conf; this.dispatcher = dispatcher; this.launchContext = launchContext; - this.container = container; + this.containerTokenIdentifier = containerTokenIdentifier; + this.containerId = containerTokenIdentifier.getContainerID(); + this.resource = containerTokenIdentifier.getResource(); this.diagnostics = new StringBuilder(); this.credentials = creds; this.metrics = metrics; - user = identifier.getApplicationSubmitter(); + user = containerTokenIdentifier.getApplicationSubmitter(); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -366,7 +371,7 @@ public ContainerLaunchContext getLaunchContext() { public ContainerStatus cloneAndGetContainerStatus() { this.readLock.lock(); try { - return BuilderUtils.newContainerStatus(this.container.getId(), + return BuilderUtils.newContainerStatus(this.containerId, getCurrentState(), diagnostics.toString(), exitCode); } finally { this.readLock.unlock(); @@ -374,10 +379,30 @@ public ContainerStatus cloneAndGetContainerStatus() { } @Override - public org.apache.hadoop.yarn.api.records.Container getContainer() { + public ContainerId getContainerId() { this.readLock.lock(); try { - return this.container; + return this.containerId; + } finally { + this.readLock.unlock(); + } + } + + @Override + public Resource getResource() { + this.readLock.lock(); + try { + return this.resource; + } finally { + this.readLock.unlock(); + } + } + + @Override + public ContainerTokenIdentifier getContainerTokenIdentifier() { + this.readLock.lock(); + try { + return this.containerTokenIdentifier; } finally { this.readLock.unlock(); } @@ -385,15 +410,15 @@ public ContainerStatus cloneAndGetContainerStatus() { @SuppressWarnings({"fallthrough", "unchecked"}) private void finished() { - ContainerId containerID = this.container.getId(); + ApplicationId applicationId = + containerId.getApplicationAttemptId().getApplicationId(); switch (getContainerState()) { case EXITED_WITH_SUCCESS: metrics.endRunningContainer(); metrics.completedContainer(); NMAuditLogger.logSuccess(user, AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl", - containerID.getApplicationAttemptId() - .getApplicationId(), containerID); + applicationId, containerId); break; case EXITED_WITH_FAILURE: metrics.endRunningContainer(); @@ -403,8 +428,7 @@ private void finished() { NMAuditLogger.logFailure(user, AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl", "Container failed with state: " + getContainerState(), - containerID.getApplicationAttemptId() - .getApplicationId(), containerID); + applicationId, containerId); break; case CONTAINER_CLEANEDUP_AFTER_KILL: metrics.endRunningContainer(); @@ -413,21 +437,21 @@ private void finished() { metrics.killedContainer(); NMAuditLogger.logSuccess(user, AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl", - containerID.getApplicationAttemptId().getApplicationId(), - containerID); + applicationId, + containerId); } - metrics.releaseContainer(this.container.getResource()); + metrics.releaseContainer(this.resource); // Inform the application @SuppressWarnings("rawtypes") EventHandler eventHandler = dispatcher.getEventHandler(); - eventHandler.handle(new ApplicationContainerFinishedEvent(containerID)); + eventHandler.handle(new ApplicationContainerFinishedEvent(containerId)); // Remove the container from the resource-monitor - eventHandler.handle(new ContainerStopMonitoringEvent(containerID)); + eventHandler.handle(new ContainerStopMonitoringEvent(containerId)); // Tell the logService too eventHandler.handle(new LogHandlerContainerFinishedEvent( - containerID, exitCode)); + containerId, exitCode)); } @SuppressWarnings("unchecked") // dispatcher not typed @@ -489,7 +513,7 @@ public ContainerState transition(ContainerImpl container, for (Map.Entry service : csd.entrySet()) { container.dispatcher.getEventHandler().handle( new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT, - container.user, container.container.getId() + container.user, container.containerId .getApplicationAttemptId().getApplicationId(), service.getKey().toString(), service.getValue())); } @@ -574,7 +598,7 @@ public ContainerState transition(ContainerImpl container, container.pendingResources.remove(rsrcEvent.getResource()); if (null == syms) { LOG.warn("Localized unknown resource " + rsrcEvent.getResource() + - " for container " + container.container.getId()); + " for container " + container.containerId); assert false; // fail container? return ContainerState.LOCALIZING; @@ -602,14 +626,14 @@ public void transition(ContainerImpl container, ContainerEvent event) { // Inform the ContainersMonitor to start monitoring the container's // resource usage. long pmemBytes = - container.container.getResource().getMemory() * 1024 * 1024L; + container.getResource().getMemory() * 1024 * 1024L; float pmemRatio = container.daemonConf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); long vmemBytes = (long) (pmemRatio * pmemBytes); container.dispatcher.getEventHandler().handle( - new ContainerStartMonitoringEvent(container.container.getId(), + new ContainerStartMonitoringEvent(container.containerId, vmemBytes, pmemBytes)); container.metrics.runningContainer(); } @@ -743,7 +767,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { container.pendingResources.remove(rsrcEvent.getResource()); if (null == syms) { LOG.warn("Localized unknown resource " + rsrcEvent.getResource() + - " for container " + container.container.getId()); + " for container " + container.containerId); assert false; // fail container? return; @@ -848,7 +872,7 @@ public void handle(ContainerEvent event) { public String toString() { this.readLock.lock(); try { - return ConverterUtils.toString(container.getId()); + return ConverterUtils.toString(this.containerId); } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d5142a1..0b5bc69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -22,8 +22,8 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import java.io.DataOutputStream; -import java.io.IOException; import java.io.File; +import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; @@ -53,12 +53,12 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; @@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; import org.apache.hadoop.yarn.util.Apps; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; public class ContainerLaunch implements Callable { @@ -86,6 +85,7 @@ private final Application app; private final Container container; private final Configuration conf; + private final Context context; private volatile AtomicBoolean shouldLaunchContainer = new AtomicBoolean(false); private volatile AtomicBoolean completed = new AtomicBoolean(false); @@ -97,9 +97,10 @@ private final LocalDirsHandlerService dirsHandler; - public ContainerLaunch(Configuration configuration, Dispatcher dispatcher, - ContainerExecutor exec, Application app, Container container, - LocalDirsHandlerService dirsHandler) { + public ContainerLaunch(Context context, Configuration configuration, + Dispatcher dispatcher, ContainerExecutor exec, Application app, + Container container, LocalDirsHandlerService dirsHandler) { + this.context = context; this.conf = configuration; this.app = app; this.exec = exec; @@ -120,7 +121,7 @@ public Integer call() { final ContainerLaunchContext launchContext = container.getLaunchContext(); final Map> localResources = container.getLocalizedResources(); - ContainerId containerID = container.getContainer().getId(); + ContainerId containerID = container.getContainerId(); String containerIdStr = ConverterUtils.toString(containerID); final List command = launchContext.getCommands(); int ret = -1; @@ -301,7 +302,7 @@ public Integer call() { * @throws IOException */ public void cleanupContainer() throws IOException { - ContainerId containerId = container.getContainer().getId(); + ContainerId containerId = container.getContainerId(); String containerIdStr = ConverterUtils.toString(containerId); LOG.info("Cleaning up container " + containerIdStr); @@ -372,7 +373,7 @@ public void cleanupContainer() throws IOException { */ private String getContainerPid(Path pidFilePath) throws Exception { String containerIdStr = - ConverterUtils.toString(container.getContainer().getId()); + ConverterUtils.toString(container.getContainerId()); String processId = null; LOG.debug("Accessing pid for container " + containerIdStr + " from pid file " + pidFilePath); @@ -550,16 +551,16 @@ public void sanitizeEnv(Map environment, */ environment.put(Environment.CONTAINER_ID.name(), container - .getContainer().getId().toString()); + .getContainerId().toString()); environment.put(Environment.NM_PORT.name(), - String.valueOf(container.getContainer().getNodeId().getPort())); + String.valueOf(this.context.getNodeId().getPort())); - environment.put(Environment.NM_HOST.name(), container.getContainer() - .getNodeId().getHost()); + environment.put(Environment.NM_HOST.name(), this.context.getNodeId() + .getHost()); - environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer() - .getNodeHttpAddress().split(":")[1]); + environment.put(Environment.NM_HTTP_PORT.name(), + String.valueOf(this.context.getHttpPort())); environment.put(Environment.LOCAL_DIRS.name(), StringUtils.join(",", appDirs)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java index 22cffb9..0fb963a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java @@ -36,8 +36,8 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; @@ -111,15 +111,16 @@ public void stop() { public void handle(ContainersLauncherEvent event) { // TODO: ContainersLauncher launches containers one by one!! Container container = event.getContainer(); - ContainerId containerId = container.getContainer().getId(); + ContainerId containerId = container.getContainerId(); switch (event.getType()) { case LAUNCH_CONTAINER: Application app = context.getApplications().get( containerId.getApplicationAttemptId().getApplicationId()); - ContainerLaunch launch = new ContainerLaunch(getConfig(), dispatcher, - exec, app, event.getContainer(), dirsHandler); + ContainerLaunch launch = + new ContainerLaunch(context, getConfig(), dispatcher, exec, app, + event.getContainer(), dirsHandler); running.put(containerId, new RunningContainer(containerLauncher.submit(launch), launch)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index abd8044..a44a99d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -359,14 +359,14 @@ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); LocalizerContext ctxt = new LocalizerContext( - c.getUser(), c.getContainer().getId(), c.getCredentials()); + c.getUser(), c.getContainerId(), c.getCredentials()); Map> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainer().getId().getApplicationAttemptId() + c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); @@ -396,21 +396,21 @@ private void handleCleanupContainerResources( for (Map.Entry> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), - c.getContainer().getId().getApplicationAttemptId() + c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceReleaseEvent(req, - c.getContainer().getId())); + c.getContainerId())); } } - String locId = ConverterUtils.toString(c.getContainer().getId()); + String locId = ConverterUtils.toString(c.getContainerId()); localizerTracker.cleanupPrivLocalizers(locId); // Delete the container directories String userName = c.getUser(); String containerIDStr = c.toString(); String appIDStr = ConverterUtils.toString( - c.getContainer().getId().getApplicationAttemptId().getApplicationId()); + c.getContainerId().getApplicationAttemptId().getApplicationId()); for (String localDir : dirsHandler.getLocalDirs()) { // Delete the user-owned container-dir @@ -430,7 +430,7 @@ private void handleCleanupContainerResources( } dispatcher.getEventHandler().handle( - new ContainerEvent(c.getContainer().getId(), + new ContainerEvent(c.getContainerId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java index 3128217..1303d32 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java @@ -43,6 +43,7 @@ private final Context nmContext; private final NMWebApp nmWebApp; private WebApp webApp; + private int port; public WebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager aclsManager, @@ -66,7 +67,7 @@ public synchronized void start() { this.webApp = WebApps.$for("node", Context.class, this.nmContext, "ws") .at(bindAddress).with(getConfig()).start(this.nmWebApp); - int port = this.webApp.httpServer().getPort(); + this.port = this.webApp.httpServer().getPort(); String webAddress = StringUtils.split(bindAddress, ':')[0] + ":" + port; getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddress); } catch (Exception e) { @@ -77,6 +78,10 @@ public synchronized void start() { super.start(); } + public int getPort() { + return this.port; + } + @Override public synchronized void stop() { if (this.webApp != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 379d998..85d2b82 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -60,7 +59,7 @@ public ContainerInfo(final Context nmContext, final Container container) { public ContainerInfo(final Context nmContext, final Container container, String requestUri, String pathPrefix) { - this.id = container.getContainer().getId().toString(); + this.id = container.getContainerId().toString(); this.nodeId = nmContext.getNodeId().toString(); ContainerStatus containerData = container.cloneAndGetContainerStatus(); this.exitCode = containerData.getExitStatus(); @@ -74,7 +73,7 @@ public ContainerInfo(final Context nmContext, final Container container, } this.user = container.getUser(); - Resource res = container.getContainer().getResource(); + Resource res = container.getResource(); if (res != null) { this.totalMemoryNeededMB = res.getMemory(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index 4a2eb38..63b0fd4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -20,7 +20,6 @@ import static org.junit.Assert.fail; -import java.io.IOException; import java.util.Collection; import org.apache.commons.logging.Log; @@ -54,7 +53,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.util.BuilderUtils; public class DummyContainerManager extends ContainerManagerImpl { @@ -94,10 +92,10 @@ public void handle(LocalizationEvent event) { .getRequestedResources().values()) { for (LocalResourceRequest req : rc) { LOG.info("DEBUG: " + req + ":" - + rsrcReqs.getContainer().getContainer().getId()); + + rsrcReqs.getContainer().getContainerId()); dispatcher.getEventHandler().handle( new ContainerResourceLocalizedEvent(rsrcReqs.getContainer() - .getContainer().getId(), req, new Path("file:///local" + .getContainerId(), req, new Path("file:///local" + req.getPath().toUri().getPath()))); } } @@ -107,7 +105,7 @@ public void handle(LocalizationEvent event) { ((ContainerLocalizationEvent) event).getContainer(); // TODO: delete the container dir this.dispatcher.getEventHandler().handle( - new ContainerEvent(container.getContainer().getId(), + new ContainerEvent(container.getContainerId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP)); break; case DESTROY_APPLICATION_RESOURCES: @@ -136,7 +134,7 @@ protected ContainersLauncher createContainersLauncher(Context context, @Override public void handle(ContainersLauncherEvent event) { Container container = event.getContainer(); - ContainerId containerId = container.getContainer().getId(); + ContainerId containerId = container.getContainerId(); switch (event.getType()) { case LAUNCH_CONTAINER: dispatcher.getEventHandler().handle( @@ -183,25 +181,15 @@ public void setBlockNewContainerRequests(boolean blockNewContainerRequests) { @Override protected void authorizeRequest(String containerIDStr, ContainerLaunchContext launchContext, - org.apache.hadoop.yarn.api.records.Container container, UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId) throws YarnRemoteException { // do Nothing } @Override - protected ContainerTokenIdentifier getContainerTokenIdentifier( - UserGroupInformation remoteUgi, - org.apache.hadoop.yarn.api.records.Container container) - throws YarnRemoteException { - try { - return new ContainerTokenIdentifier(container.getId(), - container.getNodeHttpAddress(), remoteUgi.getUserName(), - container.getResource(), System.currentTimeMillis() + 100000l, 123, - BuilderUtils.newContainerTokenIdentifier( - container.getContainerToken()).getRMIdentifer()); - } catch (IOException e) { - throw new YarnRemoteException(e); - } + protected ContainerTokenIdentifier + getContainerTokenIdentifier(UserGroupInformation remoteUgi, + ContainerTokenIdentifier containerTokenId) throws YarnRemoteException { + return containerTokenId; } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 80e3c21..5bc6eee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -18,9 +18,6 @@ package org.apache.hadoop.yarn.server.nodemanager; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.File; import java.io.IOException; @@ -30,7 +27,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -135,10 +131,7 @@ public long getRMIdentifier() { applicationAttemptId.setApplicationId(applicationId); applicationAttemptId.setAttemptId(0); cID.setApplicationAttemptId(applicationAttemptId); - Container mockContainer = mock(Container.class); - when(mockContainer.getId()).thenReturn(cID); Resource r = BuilderUtils.newResource(1024, 1); - when(mockContainer.getResource()).thenReturn(r); String user = "testing"; String host = "127.0.0.1"; int port = 1234; @@ -146,11 +139,10 @@ public long getRMIdentifier() { BuilderUtils.newContainerToken(cID, host, port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), SIMULATED_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(launchContext); - request.setContainer(mockContainer); + request.setContainerToken(containerToken); containerManager.startContainer(request); BaseContainerManagerTest.waitForContainerState(containerManager, cID, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index a96d6ee..f09f5a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -185,10 +185,7 @@ private int runAndBlock(ContainerId cId, String ... cmd) throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - org.apache.hadoop.yarn.api.records.Container containerAPI = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(container.getContainer()).thenReturn(containerAPI); - when(container.getContainer().getId()).thenReturn(cId); + when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(context.getEnvironment()).thenReturn(env); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index d41191b..62bfb39 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -34,6 +34,7 @@ import java.util.List; import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -108,10 +109,7 @@ public void testContainerLaunch() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - org.apache.hadoop.yarn.api.records.Container containerAPI = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(container.getContainer()).thenReturn(containerAPI); - when(container.getContainer().getId()).thenReturn(cId); + when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); @@ -229,10 +227,7 @@ public void testContainerLaunchError() throws IOException { ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - org.apache.hadoop.yarn.api.records.Container containerAPI = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(container.getContainer()).thenReturn(containerAPI); - when(container.getContainer().getId()).thenReturn(cId); + when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.toString()).thenReturn(containerId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index ad2793e..9b46c22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -112,9 +112,6 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, Records.newRecord(ContainerLaunchContext.class); // Construct the Container-id ContainerId cId = createContainerId(); - org.apache.hadoop.yarn.api.records.Container mockContainer = - Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class); - mockContainer.setId(cId); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -136,20 +133,16 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, containerLaunchContext.setCommands(commands); Resource resource = Records.newRecord(Resource.class); resource.setMemory(1024); - mockContainer.setResource(resource); NodeId nodeId = BuilderUtils.newNodeId("127.0.0.1", 12345); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(), user, resource, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), 0); - mockContainer.setContainerToken(containerToken); - mockContainer.setNodeHttpAddress("127.0.0.1"); - mockContainer.setNodeId(nodeId); final StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); final UserGroupInformation currentUser = UserGroupInformation .createRemoteUser(cId.toString()); currentUser.doAs(new PrivilegedExceptionAction() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index 320cf43..a550268 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -43,7 +42,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.NMNotYetReadyException; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -281,12 +279,10 @@ public void run() { try { while (!isStopped && numContainers < 10) { ContainerId cId = TestNodeManagerShutdown.createContainerId(); - Container container = - BuilderUtils.newContainer(cId, null, null, null, null, null); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(container); + startRequest.setContainerToken(null); System.out.println("no. of containers to be launched: " + numContainers); numContainers++; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index d5fe2f2..c60e3d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -56,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -158,13 +156,8 @@ public static void startContainer(NodeManager nm, ContainerId cId, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = new ContainerPBImpl(); - - mockContainer.setId(cId); NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234); - mockContainer.setNodeId(nodeId); - mockContainer.setNodeHttpAddress("localhost:12345"); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -184,16 +177,14 @@ public static void startContainer(NodeManager nm, ContainerId cId, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource resource = BuilderUtils.newResource(1024, 1); - mockContainer.setResource(resource); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(), user, resource, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), 0); - mockContainer.setContainerToken(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); UserGroupInformation currentUser = UserGroupInformation .createRemoteUser(cId.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 6c2f8c9..7ea9fcd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceTracker; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; @@ -189,8 +190,6 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) nodeStatus.setResponseId(heartBeatID++); Map> appToContainers = getAppToContainerStatusMap(nodeStatus.getContainersStatuses()); - org.apache.hadoop.yarn.api.records.Container mockContainer = - mock(org.apache.hadoop.yarn.api.records.Container.class); if (heartBeatID == 1) { Assert.assertEquals(0, nodeStatus.getContainersStatuses().size()); @@ -201,12 +200,17 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) firstContainerID.setId(heartBeatID); ContainerLaunchContext launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - when(mockContainer.getId()).thenReturn(firstContainerID); Resource resource = BuilderUtils.newResource(2, 1); - when(mockContainer.getResource()).thenReturn(resource); + long currentTime = System.currentTimeMillis(); + String user = "testUser"; + ContainerTokenIdentifier containerToken = + BuilderUtils.newContainerTokenIdentifier(BuilderUtils + .newContainerToken(firstContainerID, "127.0.0.1", 1234, user, + resource, currentTime + 10000, 123, "password".getBytes(), + currentTime)); Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, - mockContainer, null, mockMetrics, null); + new ContainerImpl(conf, mockDispatcher, launchContext, null, + mockMetrics, containerToken); this.context.getContainers().put(firstContainerID, container); } else if (heartBeatID == 2) { // Checks on the RM end @@ -227,12 +231,17 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) secondContainerID.setId(heartBeatID); ContainerLaunchContext launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - when(mockContainer.getId()).thenReturn(secondContainerID); + long currentTime = System.currentTimeMillis(); + String user = "testUser"; Resource resource = BuilderUtils.newResource(3, 1); - when(mockContainer.getResource()).thenReturn(resource); + ContainerTokenIdentifier containerToken = + BuilderUtils.newContainerTokenIdentifier(BuilderUtils + .newContainerToken(secondContainerID, "127.0.0.1", 1234, user, + resource, currentTime + 10000, 123, + "password".getBytes(), currentTime)); Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, - mockContainer, null, mockMetrics, null); + new ContainerImpl(conf, mockDispatcher, launchContext, null, + mockMetrics, containerToken); this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 2e60784..1dd8cc2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -33,7 +33,6 @@ import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -94,9 +93,14 @@ public BaseContainerManagerTest() throws UnsupportedFileSystemException { protected static Log LOG = LogFactory .getLog(BaseContainerManagerTest.class); + protected static final int HTTP_PORT = 5412; protected Configuration conf = new YarnConfiguration(); protected Context context = new NMContext(new NMContainerTokenSecretManager( - conf)); + conf)) { + public int getHttpPort() { + return HTTP_PORT; + }; + }; protected ContainerExecutor exec; protected DeletionService delSrvc; protected String user = "nobody"; @@ -177,9 +181,8 @@ public void setup() throws IOException { @Override protected void authorizeRequest(String containerIDStr, - ContainerLaunchContext launchContext, Container container, - UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId) - throws YarnRemoteException { + ContainerLaunchContext launchContext, UserGroupInformation remoteUgi, + ContainerTokenIdentifier tokenId) throws YarnRemoteException { // do nothing } }; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 858c44b..0a94154 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -18,9 +18,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -43,7 +40,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -144,24 +140,16 @@ public void testContainerSetup() throws IOException, InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - Container mockContainer = mock(Container.class); - when(mockContainer.getId()).thenReturn(cId); Resource r = BuilderUtils.newResource(512, 1); - when(mockContainer.getResource()).thenReturn(r); - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); int port = 12345; - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); @@ -245,23 +233,16 @@ public void testContainerLaunchAndStop() throws IOException, containerLaunchContext.setLocalResources(localResources); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - Container mockContainer = mock(Container.class); - when(mockContainer.getId()).thenReturn(cId); Resource r = BuilderUtils.newResource(100, 1); - when(mockContainer.getResource()).thenReturn(r); // MB - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); int port = 12345; - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); int timeoutSecs = 0; @@ -360,22 +341,15 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, containerLaunchContext.setLocalResources(localResources); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - Container mockContainer = mock(Container.class); - when(mockContainer.getId()).thenReturn(cId); Resource r = BuilderUtils.newResource(100, 1); - when(mockContainer.getResource()).thenReturn(r); // MB int port = 12345; - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, @@ -455,24 +429,16 @@ public void testLocalFilesCleanup() throws InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - Container mockContainer = mock(Container.class); - when(mockContainer.getId()).thenReturn(cId); Resource r = BuilderUtils.newResource(100, 1); - when(mockContainer.getResource()).thenReturn(r); - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); int port = 12345; - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); -// containerLaunchContext.command = new ArrayList(); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(containerLaunchContext); - request.setContainer(mockContainer); + request.setContainerToken(containerToken); containerManager.startContainer(request); BaseContainerManagerTest.waitForContainerState(containerManager, cId, @@ -551,8 +517,6 @@ public void testContainerLaunchFromPreviousRM() throws IOException, .setLocalResources(new HashMap()); Resource mockResource = BuilderUtils.newResource(1024, 1); - Container mockContainer1 = mock(Container.class); - when(mockContainer1.getId()).thenReturn(cId1); // Construct the Container with Invalid RMIdentifier StartContainerRequest startRequest1 = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -562,8 +526,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException, BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, System.currentTimeMillis() + 10000, 123, "password".getBytes(), (long) ResourceManagerConstants.RM_INVALID_IDENTIFIER); - when(mockContainer1.getContainerToken()).thenReturn(containerToken1); - startRequest1.setContainer(mockContainer1); + startRequest1.setContainerToken(containerToken1); boolean catchException = false; try { containerManager.startContainer(startRequest1); @@ -581,10 +544,6 @@ public void testContainerLaunchFromPreviousRM() throws IOException, Assert.assertTrue(catchException); // Construct the Container with a RMIdentifier within current RM - Container mockContainer2 = mock(Container.class); - when(mockContainer2.getId()).thenReturn(cId2); - - when(mockContainer2.getResource()).thenReturn(mockResource); StartContainerRequest startRequest2 = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest2.setContainerLaunchContext(containerLaunchContext); @@ -592,9 +551,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException, BuilderUtils.newContainerToken(cId1, host, port, user, mockResource, System.currentTimeMillis() + 10000, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer2.getContainerToken()).thenReturn(containerToken2); - - startRequest2.setContainer(mockContainer2); + startRequest2.setContainerToken(containerToken2); boolean noException = true; try { containerManager.startContainer(startRequest2); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java index f29bcd9..6081b9a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java @@ -91,7 +91,7 @@ public void testApplicationInit1() { for (int i = 0; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainer().getId()))); + .getContainerId()))); } } finally { if (wa != null) @@ -116,7 +116,7 @@ public void testApplicationInit2() { assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState()); verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(0) - .getContainer().getId()))); + .getContainerId()))); wa.initContainer(1); wa.initContainer(2); @@ -126,7 +126,7 @@ public void testApplicationInit2() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerInitMatcher(wa.containers.get(i) - .getContainer().getId()))); + .getContainerId()))); } } finally { if (wa != null) @@ -241,7 +241,7 @@ public void testAppFinishedOnRunningContainers() { for (int i = 1; i < wa.containers.size(); i++) { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(i) - .getContainer().getId()))); + .getContainerId()))); } wa.containerFinished(1); @@ -267,7 +267,7 @@ public void testAppFinishedOnRunningContainers() { wa.appResourcesCleanedup(); for ( Container container : wa.containers) { Assert.assertTrue(wa.context.getContainerTokenSecretManager() - .isValidStartContainerRequest(container.getContainer().getId())); + .isValidStartContainerRequest(container.getContainerId())); } assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState()); @@ -307,7 +307,7 @@ public void testAppFinishedOnCompletedContainers() { wa.appResourcesCleanedup(); for ( Container container : wa.containers) { Assert.assertTrue(wa.context.getContainerTokenSecretManager() - .isValidStartContainerRequest(container.getContainer().getId())); + .isValidStartContainerRequest(container.getContainerId())); } assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState()); } finally { @@ -370,7 +370,7 @@ public void testAppFinishedOnIniting() { verify(wa.containerBus).handle( argThat(new ContainerKillMatcher(wa.containers.get(0) - .getContainer().getId()))); + .getContainerId()))); assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT, wa.app.getApplicationState()); @@ -487,10 +487,10 @@ public boolean matches(Object argument) { containers.add(container); long currentTime = System.currentTimeMillis(); context.getContainerTokenSecretManager().startContainerSuccessful( - new ContainerTokenIdentifier(container.getContainer().getId(), "", + new ContainerTokenIdentifier(container.getContainerId(), "", "", null, currentTime + 1000, masterKey.getKeyId(), currentTime)); Assert.assertFalse(context.getContainerTokenSecretManager() - .isValidStartContainerRequest(container.getContainer().getId())); + .isValidStartContainerRequest(container.getContainerId())); } dispatcher.start(); @@ -522,7 +522,7 @@ public void initContainer(int containerNum) { public void containerFinished(int containerNum) { app.handle(new ApplicationContainerFinishedEvent(containers.get( - containerNum).getContainer().getId())); + containerNum).getContainerId())); drainDispatcherEvents(); } @@ -549,10 +549,7 @@ private Container createMockedContainer(ApplicationId appId, int containerId) { BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId); Container c = mock(Container.class); - org.apache.hadoop.yarn.api.records.Container containerAPI = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(c.getContainer()).thenReturn(containerAPI); - when(c.getContainer().getId()).thenReturn(cId); + when(c.getContainerId()).thenReturn(cId); ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class); when(c.getLaunchContext()).thenReturn(launchContext); when(launchContext.getApplicationACLs()).thenReturn( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 946461a..5e4662d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -377,7 +376,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainersLauncherEvent evt = (ContainersLauncherEvent) o; return evt.getType() == ContainersLauncherEventType.LAUNCH_CONTAINER - && wcf.cId == evt.getContainer().getContainer().getId(); + && wcf.cId == evt.getContainer().getContainerId(); } }; verify(wc.launcherBus).handle(argThat(matchesLaunchReq)); @@ -526,13 +525,6 @@ public boolean matches(Object o) { return serviceData; } - private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx, - org.apache.hadoop.yarn.api.records.Container container, - ContainerTokenIdentifier identifier) throws IOException { - return new ContainerImpl(conf, disp, ctx, container, null, metrics, - identifier); - } - @SuppressWarnings("unchecked") private class WrappedContainer { final DrainDispatcher dispatcher; @@ -612,7 +604,7 @@ private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx, } when(ctxt.getServiceData()).thenReturn(serviceData); - c = newContainer(dispatcher, ctxt, mockContainer, identifier); + c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier); dispatcher.start(); } @@ -649,7 +641,7 @@ public void initContainer() { Path p = new Path(cache, rsrc.getKey()); localPaths.put(p, Arrays.asList(rsrc.getKey())); // rsrc copied to p - c.handle(new ContainerResourceLocalizedEvent(c.getContainer().getId(), + c.handle(new ContainerResourceLocalizedEvent(c.getContainerId(), req, p)); } drainDispatcherEvents(); @@ -672,8 +664,8 @@ public void failLocalizeSpecificResource(String rsrcKey) LocalResource rsrc = localResources.get(rsrcKey); LocalResourceRequest req = new LocalResourceRequest(rsrc); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainer() - .getId(), req, e.getMessage())); + c.handle(new ContainerResourceFailedEvent(c.getContainerId(), req, e + .getMessage())); drainDispatcherEvents(); } @@ -688,7 +680,7 @@ public void failLocalizeResources(int failRsrcCount) ++counter; LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); Exception e = new Exception("Fake localization error"); - c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(), + c.handle(new ContainerResourceFailedEvent(c.getContainerId(), req, e.getMessage())); } drainDispatcherEvents(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index a5656a5..00e4fc5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.File; @@ -47,7 +45,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -163,7 +160,6 @@ public void testContainerEnvVariables() throws Exception { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = mock(Container.class); // ////// Construct the Container-id ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class); appId.setClusterTimestamp(0); @@ -176,11 +172,6 @@ public void testContainerEnvVariables() throws Exception { recordFactory.newRecordInstance(ContainerId.class); int port = 12345; cId.setApplicationAttemptId(appAttemptId); - when(mockContainer.getId()).thenReturn(cId); - - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); Map userSetEnv = new HashMap(); userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); @@ -244,15 +235,13 @@ public void testContainerEnvVariables() throws Exception { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(1024, 1); - when(mockContainer.getResource()).thenReturn(r); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 1234, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); int timeoutSecs = 0; @@ -275,27 +264,20 @@ public void testContainerEnvVariables() throws Exception { BufferedReader reader = new BufferedReader(new FileReader(processStartFile)); Assert.assertEquals(cId.toString(), reader.readLine()); - Assert.assertEquals(mockContainer.getNodeId().getHost(), - reader.readLine()); - Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), - reader.readLine()); - Assert.assertEquals( - String.valueOf(mockContainer.getNodeHttpAddress().split(":")[1]), - reader.readLine()); + Assert.assertEquals(context.getNodeId().getHost(), reader.readLine()); + Assert.assertEquals(String.valueOf(context.getNodeId().getPort()), + reader.readLine()); + Assert.assertEquals(String.valueOf(HTTP_PORT), reader.readLine()); Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine()); Assert.assertEquals(cId.toString(), containerLaunchContext .getEnvironment().get(Environment.CONTAINER_ID.name())); - Assert.assertEquals(mockContainer.getNodeId().getHost(), - containerLaunchContext.getEnvironment() - .get(Environment.NM_HOST.name())); - Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()), - containerLaunchContext.getEnvironment().get( - Environment.NM_PORT.name())); - Assert.assertEquals( - mockContainer.getNodeHttpAddress().split(":")[1], - containerLaunchContext.getEnvironment().get( - Environment.NM_HTTP_PORT.name())); + Assert.assertEquals(context.getNodeId().getHost(), containerLaunchContext + .getEnvironment().get(Environment.NM_HOST.name())); + Assert.assertEquals(String.valueOf(context.getNodeId().getPort()), + containerLaunchContext.getEnvironment().get(Environment.NM_PORT.name())); + Assert.assertEquals(String.valueOf(HTTP_PORT), containerLaunchContext + .getEnvironment().get(Environment.NM_HTTP_PORT.name())); Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext .getEnvironment().get(Environment.LOCAL_DIRS.name())); // Get the pid of the process @@ -337,7 +319,6 @@ public void testContainerEnvVariables() throws Exception { public void testDelayedKill() throws Exception { containerManager.start(); - Container mockContainer = mock(Container.class); // ////// Construct the Container-id ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class); appId.setClusterTimestamp(1); @@ -376,11 +357,7 @@ public void testDelayedKill() throws Exception { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - when(mockContainer.getId()).thenReturn(cId); - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); int port = 12345; - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); // upload the script file so that the container can run it URL resource_alpha = @@ -403,15 +380,13 @@ public void testDelayedKill() throws Exception { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(1024, 1); - when(mockContainer.getResource()).thenReturn(r); ContainerToken containerToken = BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); int timeoutSecs = 0; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index cd12ea5..3626626 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -495,7 +495,7 @@ public boolean matches(Object o) { Thread.sleep(1000); dispatcher.await(); String appStr = ConverterUtils.toString(appId); - String ctnrStr = c.getContainer().getId().toString(); + String ctnrStr = c.getContainerId().toString(); ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class); verify(exec).startLocalizer(tokenPathCaptor.capture(), isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr), @@ -571,7 +571,7 @@ public boolean matches(Object o) { public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED - && c.getContainer().getId() == evt.getContainerID(); + && c.getContainerId() == evt.getContainerID(); } }; // total 2 resource localzation calls. one for each resource. @@ -760,11 +760,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception { // Container - 1 ContainerImpl container1 = createMockContainer(user, 1); - String localizerId1 = container1.getContainer().getId().toString(); + String localizerId1 = container1.getContainerId().toString(); rls.getPrivateLocalizers().put( localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1 - .getContainer().getId(), null), localizerId1)); + .getContainerId(), null), localizerId1)); LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1); dispatcher1.getEventHandler().handle( @@ -775,11 +775,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception { // Container - 2 now makes the request. ContainerImpl container2 = createMockContainer(user, 2); - String localizerId2 = container2.getContainer().getId().toString(); + String localizerId2 = container2.getContainerId().toString(); rls.getPrivateLocalizers().put( localizerId2, rls.new LocalizerRunner(new LocalizerContext(user, container2 - .getContainer().getId(), null), localizerId2)); + .getContainerId(), null), localizerId2)); LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2); dispatcher1.getEventHandler().handle( createContainerLocalizationEvent(container2, @@ -920,11 +920,11 @@ public void testLocalResourcePath() throws Exception { // Container - 1 Container container1 = createMockContainer(user, 1); - String localizerId1 = container1.getContainer().getId().toString(); + String localizerId1 = container1.getContainerId().toString(); rls.getPrivateLocalizers().put( localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1 - .getContainer().getId(), null), localizerId1)); + .getContainerId(), null), localizerId1)); // Creating two requests for container // 1) Private resource @@ -1317,10 +1317,7 @@ private ContainerLocalizationRequestEvent createContainerLocalizationEvent( private ContainerImpl createMockContainer(String user, int containerId) { ContainerImpl container = mock(ContainerImpl.class); - org.apache.hadoop.yarn.api.records.Container c = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(container.getContainer()).thenReturn(c); - when(container.getContainer().getId()).thenReturn( + when(container.getContainerId()).thenReturn( BuilderUtils.newContainerId(1, 1, 1, containerId)); when(container.getUser()).thenReturn(user); Credentials mockCredentials = mock(Credentials.class); @@ -1360,11 +1357,8 @@ private static Container getMockContainer(ApplicationId appId, int id) { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id); - org.apache.hadoop.yarn.api.records.Container containerAPI = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(c.getContainer()).thenReturn(containerAPI); when(c.getUser()).thenReturn("user0"); - when(c.getContainer().getId()).thenReturn(cId); + when(c.getContainerId()).thenReturn(cId); Credentials creds = new Credentials(); creds.addToken(new Text("tok" + id), getToken(id)); when(c.getCredentials()).thenReturn(creds); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 736854d..7e5ece3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -64,10 +64,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -714,7 +714,6 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = mock(Container.class); // ////// Construct the Container-id ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class); @@ -724,8 +723,6 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0); - when(mockContainer.getId()).thenReturn(cId); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -746,15 +743,14 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1); - when(mockContainer.getResource()).thenReturn(r); - when(mockContainer.getContainerToken()).thenReturn( - BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r, - System.currentTimeMillis() + 10000L, 123, "password".getBytes(), - super.DUMMY_RM_IDENTIFIER)); + ContainerToken containerToken = + BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r, + System.currentTimeMillis() + 10000L, 123, "password".getBytes(), + super.DUMMY_RM_IDENTIFIER); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); this.containerManager.startContainer(startRequest); BaseContainerManagerTest.waitForContainerState(this.containerManager, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 06d45aa..a97f6a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.File; @@ -45,7 +44,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -202,7 +200,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = mock(Container.class); // ////// Construct the Container-id ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class); @@ -215,12 +212,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException, ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); cId.setId(0); cId.setApplicationAttemptId(appAttemptId); - when(mockContainer.getId()).thenReturn(cId); - - when(mockContainer.getNodeId()).thenReturn(context.getNodeId()); int port = 12345; - when(mockContainer.getNodeHttpAddress()).thenReturn( - context.getNodeId().getHost() + ":" + port); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -242,7 +234,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); - when(mockContainer.getResource()).thenReturn(r); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); @@ -250,8 +241,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException, BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(), port, user, r, System.currentTimeMillis() + 10000L, 123, "password".getBytes(), super.DUMMY_RM_IDENTIFIER); - when(mockContainer.getContainerToken()).thenReturn(containerToken); - startRequest.setContainer(mockContainer); + startRequest.setContainerToken(containerToken); containerManager.startContainer(startRequest); int timeoutSecs = 0; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 13949b0..5dce308 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -18,9 +18,6 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -34,10 +31,11 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; @@ -52,7 +50,7 @@ private final Map> resource = new HashMap>(); private RecordFactory recordFactory; - private org.apache.hadoop.yarn.api.records.Container mockContainer; + private final ContainerTokenIdentifier containerTokenIdentifier; public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration conf, String user, @@ -65,15 +63,12 @@ public MockContainer(ApplicationAttemptId appAttemptId, this.launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); - ContainerToken containerToken = - BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user, - BuilderUtils.newResource(1024, 1), currentTime + 10000, 123, - "password".getBytes(), currentTime); + this.containerTokenIdentifier = + BuilderUtils.newContainerTokenIdentifier(BuilderUtils + .newContainerToken(id, "127.0.0.1", 1234, user, + BuilderUtils.newResource(1024, 1), currentTime + 10000, 123, + "password".getBytes(), currentTime)); this.state = ContainerState.NEW; - - mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); - when(mockContainer.getContainerToken()).thenReturn(containerToken); - when(mockContainer.getId()).thenReturn(id); } public void setState(ContainerState state) { @@ -126,7 +121,17 @@ public void handle(ContainerEvent event) { } @Override - public org.apache.hadoop.yarn.api.records.Container getContainer() { - return this.mockContainer; + public ContainerId getContainerId() { + return this.id; + } + + @Override + public Resource getResource() { + return this.containerTokenIdentifier.getResource(); + } + + @Override + public ContainerTokenIdentifier getContainerTokenIdentifier() { + return this.containerTokenIdentifier; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index dbb6866..743d01d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -180,17 +180,13 @@ public boolean isPmemCheckEnabled() { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - org.apache.hadoop.yarn.api.records.Container mockContainer = - mock(org.apache.hadoop.yarn.api.records.Container.class); long currentTime = System.currentTimeMillis(); ContainerToken containerToken = BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); - when(mockContainer.getContainerToken()).thenReturn(containerToken); - when(mockContainer.getId()).thenReturn(containerId); Container container = - new ContainerImpl(conf, dispatcher, launchContext, mockContainer, + new ContainerImpl(conf, dispatcher, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index 5333a61..9b059bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -188,17 +188,17 @@ public void testNodeAppsNone() throws JSONException, Exception { Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); nmContext.getContainers() - .put(container1.getContainer().getId(), container1); + .put(container1.getContainerId(), container1); nmContext.getContainers() - .put(container2.getContainer().getId(), container2); + .put(container2.getContainerId(), container2); - app.getContainers().put(container1.getContainer().getId(), container1); - app.getContainers().put(container2.getContainer().getId(), container2); + app.getContainers().put(container1.getContainerId(), container1); + app.getContainers().put(container2.getContainerId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainer().getId().toString(), container1 - .getContainer().getId().toString()); - hash.put(container2.getContainer().getId().toString(), container2 - .getContainer().getId().toString()); + hash.put(container1.getContainerId().toString(), container1 + .getContainerId().toString()); + hash.put(container2.getContainerId().toString(), container2 + .getContainerId().toString()); return hash; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 5112c6c..70d65e3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -189,17 +189,17 @@ public void testNodeContainersNone() throws JSONException, Exception { Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); nmContext.getContainers() - .put(container1.getContainer().getId(), container1); + .put(container1.getContainerId(), container1); nmContext.getContainers() - .put(container2.getContainer().getId(), container2); + .put(container2.getContainerId(), container2); - app.getContainers().put(container1.getContainer().getId(), container1); - app.getContainers().put(container2.getContainer().getId(), container2); + app.getContainers().put(container1.getContainerId(), container1); + app.getContainers().put(container2.getContainerId(), container2); HashMap hash = new HashMap(); - hash.put(container1.getContainer().getId().toString(), container1 - .getContainer().getId().toString()); - hash.put(container2.getContainer().getId().toString(), container2 - .getContainer().getId().toString()); + hash.put(container1.getContainerId().toString(), container1 + .getContainerId().toString()); + hash.put(container2.getContainerId().toString(), container2 + .getContainerId().toString()); return hash; } @@ -472,7 +472,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, String state, String user, int exitCode, String diagnostics, String nodeId, int totalMemoryNeededMB, String logsLink) throws JSONException, Exception { - WebServicesTestUtils.checkStringMatch("id", cont.getContainer().getId() + WebServicesTestUtils.checkStringMatch("id", cont.getContainerId() .toString(), id); WebServicesTestUtils.checkStringMatch("state", cont.getContainerState() .toString(), state); @@ -486,7 +486,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, .toString(), nodeId); assertEquals("totalMemoryNeededMB wrong", 0, totalMemoryNeededMB); String shortLink = - ujoin("containerlogs", cont.getContainer().getId().toString(), + ujoin("containerlogs", cont.getContainerId().toString(), cont.getUser()); assertTrue("containerLogsLink wrong", logsLink.contains(shortLink)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 83e323d..69d79f3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -109,7 +109,7 @@ private void launch() throws IOException, YarnRemoteException { StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(launchContext); - request.setContainer(masterContainer); + request.setContainerToken(masterContainer.getContainerToken()); containerMgrProxy.startContainer(request); LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 841bc28..2c70c0e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -342,7 +342,7 @@ private synchronized void assign(Priority priority, NodeType type, // Launch the container StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(createCLC()); - startRequest.setContainer(container); + startRequest.setContainerToken(container.getContainerToken()); nodeManager.startContainer(startRequest); break; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 6018a03..24261c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; @@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; @@ -161,11 +163,19 @@ public void heartbeat() throws IOException, YarnRemoteException { synchronized public StartContainerResponse startContainer( StartContainerRequest request) throws YarnRemoteException { - Container requestContainer = request.getContainer(); - ApplicationId applicationId = - requestContainer.getId().getApplicationAttemptId(). - getApplicationId(); + ContainerToken containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + + ContainerId containerID = tokenId.getContainerID(); + ApplicationId applicationId = + containerID.getApplicationAttemptId().getApplicationId(); List applicationContainers = containers.get(applicationId); if (applicationContainers == null) { @@ -175,18 +185,18 @@ synchronized public StartContainerResponse startContainer( // Sanity check for (Container container : applicationContainers) { - if (container.getId().compareTo(requestContainer.getId()) + if (container.getId().compareTo(containerID) == 0) { throw new IllegalStateException( - "Container " + requestContainer.getId() + + "Container " + containerID + " already setup on node " + containerManagerAddress); } } Container container = - BuilderUtils.newContainer(requestContainer.getId(), + BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, - requestContainer.getResource(), + tokenId.getResource(), null, null // DKDC - Doesn't matter ); @@ -195,8 +205,8 @@ synchronized public StartContainerResponse startContainer( "", -1000); applicationContainers.add(container); containerStatusMap.put(container, containerStatus); - Resources.subtractFrom(available, requestContainer.getResource()); - Resources.addTo(used, requestContainer.getResource()); + Resources.subtractFrom(available, tokenId.getResource()); + Resources.addTo(used, tokenId.getResource()); if(LOG.isDebugEnabled()) { LOG.debug("startContainer:" + " node=" + containerManagerAddress diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index c7679c8..7617292 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.io.IOException; import java.util.Map; import org.apache.commons.logging.Log; @@ -33,11 +34,15 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -57,8 +62,6 @@ String attemptIdAtContainerManager = null; String containerIdAtContainerManager = null; String nmHostAtContainerManager = null; - int nmPortAtContainerManager; - int nmHttpPortAtContainerManager; long submitTimeAtContainerManager; int maxAppAttempts; @@ -70,17 +73,21 @@ launched = true; Map env = request.getContainerLaunchContext().getEnvironment(); - ContainerId containerId = - request.getContainer().getId(); + + ContainerToken containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + + ContainerId containerId = tokenId.getContainerID(); containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); - nmHostAtContainerManager = request.getContainer().getNodeId().getHost(); - nmPortAtContainerManager = - request.getContainer().getNodeId().getPort(); - nmHttpPortAtContainerManager = - Integer.parseInt(request.getContainer().getNodeHttpAddress() - .split(":")[1]); + nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = @@ -137,10 +144,6 @@ public void testAMLaunchAndCleanup() throws Exception { .toString(), containerManager.containerIdAtContainerManager); Assert.assertEquals(nm1.getNodeId().getHost(), containerManager.nmHostAtContainerManager); - Assert.assertEquals(nm1.getNodeId().getPort(), - containerManager.nmPortAtContainerManager); - Assert.assertEquals(nm1.getHttpPort(), - containerManager.nmHttpPortAtContainerManager); Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, containerManager.maxAppAttempts); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index c79e2fe..b625fe9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -375,7 +375,7 @@ public Void run() { request.setContainerLaunchContext(context); allocatedContainer.setContainerToken(BuilderUtils.newContainerToken( allocatedContainer.getNodeId(), passowrd, newTokenId)); - request.setContainer(allocatedContainer); + request.setContainerToken(allocatedContainer.getContainerToken()); //Calling startContainer with an expired token. try { @@ -540,7 +540,7 @@ void callWithIllegalContainerID(ContainerManager client, ContainerId oldContainerId = container.getId(); try { container.setId(newContainerId); - request.setContainer(container); + request.setContainerToken(container.getContainerToken()); request.setContainerLaunchContext(context); client.startContainer(request); fail("Connection initiation with unauthorized " @@ -569,7 +569,7 @@ void callWithIllegalResource(ContainerManager client, Resource rsrc = container.getResource(); container.setResource(BuilderUtils.newResource(2048, 1)); request.setContainerLaunchContext(context); - request.setContainer(container); + request.setContainerToken(container.getContainerToken()); try { client.startContainer(request); fail("Connection initiation with unauthorized " @@ -597,7 +597,7 @@ void callWithIllegalUserName(ContainerManager client, createContainerLaunchContextForTest(tokenId); String user = "invalidUser"; request.setContainerLaunchContext(context); - request.setContainer(container); + request.setContainerToken(container.getContainerToken()); try { client.startContainer(request); fail("Connection initiation with unauthorized "