diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
index 78862b9..ca161d5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
@@ -21,8 +21,8 @@
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ContainerManager;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
/**
*
The request sent by the ApplicationMaster to the
@@ -62,9 +62,9 @@
@Public
@Stable
- public Container getContainer();
+ public ContainerToken getContainerToken();
@Public
@Stable
- public void setContainer(Container container);
+ public void setContainerToken(ContainerToken container);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
index d4190e0..5073f0f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
@@ -19,14 +19,14 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerTokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProtoOrBuilder;
@@ -39,7 +39,7 @@
private ContainerLaunchContext containerLaunchContext = null;
- private Container container = null;
+ private ContainerToken containerToken = null;
public StartContainerRequestPBImpl() {
builder = StartContainerRequestProto.newBuilder();
@@ -61,8 +61,8 @@ private void mergeLocalToBuilder() {
if (this.containerLaunchContext != null) {
builder.setContainerLaunchContext(convertToProtoFormat(this.containerLaunchContext));
}
- if(this.container != null) {
- builder.setContainer(convertToProtoFormat(this.container));
+ if(this.containerToken != null) {
+ builder.setContainerToken(convertToProtoFormat(this.containerToken));
}
}
@@ -104,25 +104,25 @@ public void setContainerLaunchContext(ContainerLaunchContext containerLaunchCont
}
@Override
- public Container getContainer() {
+ public ContainerToken getContainerToken() {
StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.container != null) {
- return this.container;
+ if (this.containerToken != null) {
+ return this.containerToken;
}
- if (!p.hasContainer()) {
+ if (!p.hasContainerToken()) {
return null;
}
- this.container = convertFromProtoFormat(p.getContainer());
- return this.container;
+ this.containerToken = convertFromProtoFormat(p.getContainerToken());
+ return this.containerToken;
}
@Override
- public void setContainer(Container container) {
+ public void setContainerToken(ContainerToken containerToken) {
maybeInitBuilder();
- if(container == null) {
- builder.clearContainer();
+ if(containerToken == null) {
+ builder.clearContainerToken();
}
- this.container = container;
+ this.containerToken = containerToken;
}
private ContainerLaunchContextPBImpl convertFromProtoFormat(ContainerLaunchContextProto p) {
@@ -135,11 +135,11 @@ private ContainerLaunchContextProto convertToProtoFormat(ContainerLaunchContext
- private ContainerPBImpl convertFromProtoFormat(ContainerProto containerProto) {
- return new ContainerPBImpl(containerProto);
+ private ContainerTokenPBImpl convertFromProtoFormat(TokenProto containerProto) {
+ return new ContainerTokenPBImpl(containerProto);
}
- private ContainerProto convertToProtoFormat(Container container) {
- return ((ContainerPBImpl)container).getProto();
+ private TokenProto convertToProtoFormat(ContainerToken container) {
+ return ((ContainerTokenPBImpl)container).getProto();
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 480fe16..ed3f871 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -173,7 +173,7 @@ message GetQueueUserAclsInfoResponseProto {
message StartContainerRequestProto {
optional ContainerLaunchContextProto container_launch_context = 1;
- optional ContainerProto container = 2;
+ optional hadoop.common.TokenProto container_token = 2;
}
message StartContainerResponseProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 9f1437e..2f9f359 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -763,7 +763,7 @@ public void run() {
StartContainerRequest startReq = Records
.newRecord(StartContainerRequest.class);
startReq.setContainerLaunchContext(ctx);
- startReq.setContainer(container);
+ startReq.setContainerToken(container.getContainerToken());
try {
cm.startContainer(startReq);
} catch (YarnRemoteException e) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java
index 85c313c..2bf9c93 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java
@@ -212,7 +212,7 @@ public synchronized void stop() {
try {
StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
- startRequest.setContainer(container);
+ startRequest.setContainerToken(container.getContainerToken());
startRequest.setContainerLaunchContext(containerLaunchContext);
startResponse = containerManager.startContainer(startRequest);
if (LOG.isDebugEnabled()) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
index 5f1dafb..10d2447 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
@@ -39,11 +39,11 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -51,6 +51,7 @@
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
@@ -89,23 +90,25 @@ private void testRPCTimeout(String rpcClass) throws Exception {
server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- ContainerId containerId = recordFactory
- .newRecordInstance(ContainerId.class);
+
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId applicationAttemptId = recordFactory
- .newRecordInstance(ApplicationAttemptId.class);
- applicationAttemptId.setApplicationId(applicationId);
- applicationAttemptId.setAttemptId(0);
- containerId.setApplicationAttemptId(applicationAttemptId);
- containerId.setId(100);
- Container container =
- BuilderUtils.newContainer(containerId, null, null, recordFactory
- .newRecordInstance(Resource.class), null, null);
+ ApplicationAttemptId applicationAttemptId =
+ ApplicationAttemptId.newInstance(applicationId, 0);
+ ContainerId containerId =
+ ContainerId.newInstance(applicationAttemptId, 100);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Resource resource = Resource.newInstance(1234, 2);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "localhost", "user",
+ resource, System.currentTimeMillis() + 10000, 42, 42);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(nodeId, "password".getBytes(),
+ containerTokenIdentifier);
StartContainerRequest scRequest = recordFactory
.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
- scRequest.setContainer(container);
+ scRequest.setContainerToken(containerToken);
try {
proxy.startContainer(scRequest);
} catch (Exception e) {
@@ -138,9 +141,6 @@ public GetContainerStatusResponse getContainerStatus(
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException, IOException {
- StartContainerResponse response = recordFactory
- .newRecordInstance(StartContainerResponse.class);
- status = recordFactory.newRecordInstance(ContainerStatus.class);
try {
// make the thread sleep to look like its not going to respond
Thread.sleep(10000);
@@ -148,10 +148,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
LOG.error(e);
throw new YarnRemoteException(e);
}
- status.setState(ContainerState.RUNNING);
- status.setContainerId(request.getContainer().getId());
- status.setExitStatus(0);
- return response;
+ throw new YarnRemoteException("Shouldn't happen!!");
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index 6308658..013c9b7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn;
+import java.io.IOException;
import java.net.InetSocketAddress;
import junit.framework.Assert;
@@ -39,18 +40,21 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -111,30 +115,29 @@ private void test(String rpcClass) throws Exception {
NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ContainerId containerId =
- recordFactory.newRecordInstance(ContainerId.class);
+
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId applicationAttemptId =
- recordFactory.newRecordInstance(ApplicationAttemptId.class);
- applicationAttemptId.setApplicationId(applicationId);
- applicationAttemptId.setAttemptId(0);
- containerId.setApplicationAttemptId(applicationAttemptId);
- containerId.setId(100);
- Container mockContainer =
- BuilderUtils.newContainer(containerId, null, null, recordFactory
- .newRecordInstance(Resource.class), null, null);
-// containerLaunchContext.env = new HashMap();
-// containerLaunchContext.command = new ArrayList();
-
- StartContainerRequest scRequest =
+ ApplicationAttemptId.newInstance(applicationId, 0);
+ ContainerId containerId =
+ ContainerId.newInstance(applicationAttemptId, 100);
+ StartContainerRequest scRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
- scRequest.setContainer(mockContainer);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Resource resource = Resource.newInstance(1234, 2);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "localhost", "user",
+ resource, System.currentTimeMillis() + 10000, 42, 42);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(nodeId, "password".getBytes(),
+ containerTokenIdentifier);
+ scRequest.setContainerToken(containerToken);
proxy.startContainer(scRequest);
GetContainerStatusRequest gcsRequest =
recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(mockContainer.getId());
+ gcsRequest.setContainerId(containerId);
GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest);
ContainerStatus status = response.getStatus();
@@ -142,7 +145,7 @@ private void test(String rpcClass) throws Exception {
boolean exception = false;
try {
StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(mockContainer.getId());
+ stopRequest.setContainerId(containerId);
proxy.stopContainer(stopRequest);
} catch (YarnRemoteException e) {
exception = true;
@@ -176,11 +179,19 @@ public GetContainerStatusResponse getContainerStatus(
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
+ ContainerToken containerToken = request.getContainerToken();
+ ContainerTokenIdentifier tokenId = null;
+
+ try {
+ tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
StartContainerResponse response =
recordFactory.newRecordInstance(StartContainerResponse.class);
status = recordFactory.newRecordInstance(ContainerStatus.class);
status.setState(ContainerState.RUNNING);
- status.setContainerId(request.getContainer().getId());
+ status.setContainerId(tokenId.getContainerID());
status.setExitStatus(0);
return response;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 54fbdb5..80fcc49 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -42,6 +42,13 @@
*/
NodeId getNodeId();
+ /**
+ * Return the node http-address. Usable only after the Webserver is started.
+ *
+ * @return the http-port
+ */
+ int getHttpPort();
+
ConcurrentMap getApplications();
ConcurrentMap getContainers();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 89032f2..5b534ba 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -113,7 +113,7 @@ public int launchContainer(Container container,
List localDirs, List logDirs) throws IOException {
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
// create container dirs on all disks
String containerIdStr = ConverterUtils.toString(containerId);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 0317556..dbc66fb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -217,11 +217,11 @@ public int launchContainer(Container container,
String user, String appId, Path containerWorkDir,
List localDirs, List logDirs) throws IOException {
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerId);
resourcesHandler.preExecute(containerId,
- container.getContainer().getResource());
+ container.getResource());
String resourcesOptions = resourcesHandler.getResourcesOption(
containerId);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 85f5784..2ae578b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -32,7 +32,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -56,7 +55,6 @@
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.CompositeService;
-import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.util.Records;
import com.google.common.annotations.VisibleForTesting;
@@ -173,9 +171,10 @@ public void init(Configuration conf) {
addService(containerManager);
((NMContext) context).setContainerManager(containerManager);
- Service webServer = createWebServer(context, containerManager
+ WebServer webServer = createWebServer(context, containerManager
.getContainersMonitor(), this.aclsManager, dirsHandler);
addService(webServer);
+ ((NMContext) context).setWebServer(webServer);
dispatcher.register(ContainerManagerEventType.class, containerManager);
dispatcher.register(NodeManagerEventType.class, this);
@@ -297,6 +296,7 @@ protected void cleanupContainers(NodeManagerEventType eventType) {
private final NMContainerTokenSecretManager containerTokenSecretManager;
private ContainerManager containerManager;
+ private WebServer webServer;
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
@@ -316,6 +316,11 @@ public NodeId getNodeId() {
}
@Override
+ public int getHttpPort() {
+ return this.webServer.getPort();
+ }
+
+ @Override
public ConcurrentMap getApplications() {
return this.applications;
}
@@ -342,6 +347,10 @@ public ContainerManager getContainerManager() {
public void setContainerManager(ContainerManager containerManager) {
this.containerManager = containerManager;
}
+
+ public void setWebServer(WebServer webServer) {
+ this.webServer = webServer;
+ }
}
@@ -352,12 +361,6 @@ public NodeHealthCheckerService getNodeHealthChecker() {
return nodeHealthChecker;
}
- private void reboot() {
- LOG.info("Rebooting the node manager.");
- NodeManager nodeManager = createNewNodeManager();
- nodeManager.initAndStartNodeManager(this.getConfig(), true);
- }
-
private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) {
try {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b33e7b3..53c01d9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -155,13 +155,8 @@ public void start() {
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.getNodeId();
-
- InetSocketAddress httpBindAddress = getConfig().getSocketAddr(
- YarnConfiguration.NM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_PORT);
+ this.httpPort = this.context.getHttpPort();
try {
- this.httpPort = httpBindAddress.getPort();
// Registration has to be in start so that ContainerManager can get the
// perNM tokens needed to authenticate ContainerTokens.
registerWithRM();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 6ffaa99..89ccb87 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -52,7 +52,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -272,14 +272,13 @@ public void stop() {
}
// Get the remoteUGI corresponding to the api call.
- private UserGroupInformation getRemoteUgi(String containerIDStr)
+ private UserGroupInformation getRemoteUgi()
throws YarnRemoteException {
UserGroupInformation remoteUgi;
try {
remoteUgi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
- String msg = "Cannot obtain the user-name for containerId: "
- + containerIDStr + ". Got exception: "
+ String msg = "Cannot obtain the user-name. Got exception: "
+ StringUtils.stringifyException(e);
LOG.warn(msg);
throw RPCUtil.getRemoteException(msg);
@@ -307,7 +306,7 @@ private ContainerTokenIdentifier selectContainerTokenIdentifier(
@VisibleForTesting
protected ContainerTokenIdentifier getContainerTokenIdentifier(
UserGroupInformation remoteUgi,
- org.apache.hadoop.yarn.api.records.Container container)
+ ContainerTokenIdentifier containerTokenIdentifier)
throws YarnRemoteException {
if (UserGroupInformation.isSecurityEnabled()) {
if (LOG.isDebugEnabled()) {
@@ -317,12 +316,7 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier(
// Get the tokenId from the remote user ugi
return selectContainerTokenIdentifier(remoteUgi);
} else {
- try {
- return BuilderUtils.newContainerTokenIdentifier(container
- .getContainerToken());
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
- }
+ return containerTokenIdentifier;
}
}
@@ -341,7 +335,6 @@ protected ContainerTokenIdentifier getContainerTokenIdentifier(
@VisibleForTesting
protected void authorizeRequest(String containerIDStr,
ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
throws YarnRemoteException {
@@ -380,13 +373,6 @@ protected void authorizeRequest(String containerIDStr,
+ System.currentTimeMillis() + " found "
+ tokenId.getExpiryTimeStamp());
}
-
- Resource resource = tokenId.getResource();
- if (resource == null || !resource.equals(container.getResource())) {
- unauthorized = true;
- messageBuilder.append("\nExpected resource " + resource
- + " but found " + container.getResource());
- }
}
}
@@ -412,16 +398,23 @@ public StartContainerResponse startContainer(StartContainerRequest request)
}
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
- org.apache.hadoop.yarn.api.records.Container lauchContainer =
- request.getContainer();
- ContainerId containerID = lauchContainer.getId();
- String containerIDStr = containerID.toString();
+ ContainerToken token = request.getContainerToken();
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ ContainerTokenIdentifier tokenIdentifier = null;
+ try {
+ tokenIdentifier = BuilderUtils.newContainerTokenIdentifier(token);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
ContainerTokenIdentifier tokenId =
- getContainerTokenIdentifier(remoteUgi, lauchContainer);
- authorizeRequest(containerIDStr, launchContext, lauchContainer, remoteUgi,
- tokenId);
+ getContainerTokenIdentifier(remoteUgi, tokenIdentifier);
+
+ ContainerId containerID = tokenId.getContainerID();
+ String containerIDStr = containerID.toString();
+
+ authorizeRequest(containerIDStr, launchContext, remoteUgi, tokenId);
// Is the container coming from unknown RM
if (tokenId.getRMIdentifer() != nodeStatusUpdater
@@ -458,8 +451,9 @@ public StartContainerResponse startContainer(StartContainerRequest request)
// //////////// End of parsing credentials
String user = tokenId.getApplicationSubmitter();
- Container container = new ContainerImpl(getConfig(), this.dispatcher,
- launchContext, lauchContainer, credentials, metrics, tokenId);
+ Container container =
+ new ContainerImpl(getConfig(), this.dispatcher, launchContext,
+ credentials, metrics, tokenId);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
@@ -501,7 +495,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
// TODO launchedContainer misplaced -> doesn't necessarily mean a container
// launch. A finished Application will not launch containers.
metrics.launchedContainer();
- metrics.allocateContainer(lauchContainer.getResource());
+ metrics.allocateContainer(tokenId.getResource());
return response;
}
@@ -518,7 +512,7 @@ public StopContainerResponse stopContainer(StopContainerRequest request)
// TODO: Only the container's owner can kill containers today.
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ UserGroupInformation remoteUgi = getRemoteUgi();
Container container = this.context.getContainers().get(containerID);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -532,8 +526,8 @@ public StopContainerResponse stopContainer(StopContainerRequest request)
containerID);
return response; // Return immediately.
}
- authorizeRequest(containerIDStr, null, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainer()));
+ authorizeRequest(containerIDStr, null, remoteUgi,
+ getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
dispatcher.getEventHandler().handle(
new ContainerKillEvent(containerID,
@@ -561,15 +555,15 @@ public GetContainerStatusResponse getContainerStatus(
// TODO: Only the container's owner can get containers' status today.
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ UserGroupInformation remoteUgi = getRemoteUgi();
LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
if (container == null) {
throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " is not handled by this NodeManager");
}
- authorizeRequest(containerIDStr, null, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainer()));
+ authorizeRequest(containerIDStr, null, remoteUgi,
+ getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
index b74be63..ef9e5a4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
@@ -36,7 +36,7 @@
final Container container;
public ApplicationContainerInitEvent(Container container) {
- super(container.getContainer().getId().getApplicationAttemptId()
+ super(container.getContainerId().getApplicationAttemptId()
.getApplicationId(), ApplicationEventType.INIT_CONTAINER);
this.container = container;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index dc065a5..7a2ccb8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -273,14 +273,14 @@ public void transition(ApplicationImpl app, ApplicationEvent event) {
ApplicationContainerInitEvent initEvent =
(ApplicationContainerInitEvent) event;
Container container = initEvent.getContainer();
- app.containers.put(container.getContainer().getId(), container);
- LOG.info("Adding " + container.getContainer().getId()
+ app.containers.put(container.getContainerId(), container);
+ LOG.info("Adding " + container.getContainerId()
+ " to application " + app.toString());
switch (app.getApplicationState()) {
case RUNNING:
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainer().getId()));
+ container.getContainerId()));
break;
case INITING:
case NEW:
@@ -301,7 +301,7 @@ public void transition(ApplicationImpl app, ApplicationEvent event) {
// Start all the containers waiting for ApplicationInit
for (Container container : app.containers.values()) {
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainer().getId()));
+ container.getContainerId()));
}
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index c975101..e69e61a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -23,13 +23,20 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
public interface Container extends EventHandler {
- org.apache.hadoop.yarn.api.records.Container getContainer();
+ ContainerId getContainerId();
+
+ Resource getResource();
+
+ ContainerTokenIdentifier getContainerTokenIdentifier();
String getUser();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 8a95043..44164b2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -37,11 +37,13 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -76,7 +78,9 @@
private final Credentials credentials;
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
- private final org.apache.hadoop.yarn.api.records.Container container;
+ private final ContainerTokenIdentifier containerTokenIdentifier;
+ private final ContainerId containerId;
+ private final Resource resource;
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@@ -97,18 +101,19 @@
new ArrayList();
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
- ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
- Credentials creds, NodeManagerMetrics metrics,
- ContainerTokenIdentifier identifier) throws IOException {
+ ContainerLaunchContext launchContext, Credentials creds,
+ NodeManagerMetrics metrics,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
- this.container = container;
+ this.containerTokenIdentifier = containerTokenIdentifier;
+ this.containerId = containerTokenIdentifier.getContainerID();
+ this.resource = containerTokenIdentifier.getResource();
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
- user = identifier.getApplicationSubmitter();
+ user = containerTokenIdentifier.getApplicationSubmitter();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
@@ -366,7 +371,7 @@ public ContainerLaunchContext getLaunchContext() {
public ContainerStatus cloneAndGetContainerStatus() {
this.readLock.lock();
try {
- return BuilderUtils.newContainerStatus(this.container.getId(),
+ return BuilderUtils.newContainerStatus(this.containerId,
getCurrentState(), diagnostics.toString(), exitCode);
} finally {
this.readLock.unlock();
@@ -374,10 +379,20 @@ public ContainerStatus cloneAndGetContainerStatus() {
}
@Override
- public org.apache.hadoop.yarn.api.records.Container getContainer() {
+ public ContainerId getContainerId() {
+ return this.containerId;
+ }
+
+ @Override
+ public Resource getResource() {
+ return this.resource;
+ }
+
+ @Override
+ public ContainerTokenIdentifier getContainerTokenIdentifier() {
this.readLock.lock();
try {
- return this.container;
+ return this.containerTokenIdentifier;
} finally {
this.readLock.unlock();
}
@@ -385,15 +400,15 @@ public ContainerStatus cloneAndGetContainerStatus() {
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
- ContainerId containerID = this.container.getId();
+ ApplicationId applicationId =
+ containerId.getApplicationAttemptId().getApplicationId();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
metrics.completedContainer();
NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
- containerID.getApplicationAttemptId()
- .getApplicationId(), containerID);
+ applicationId, containerId);
break;
case EXITED_WITH_FAILURE:
metrics.endRunningContainer();
@@ -403,8 +418,7 @@ private void finished() {
NMAuditLogger.logFailure(user,
AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
"Container failed with state: " + getContainerState(),
- containerID.getApplicationAttemptId()
- .getApplicationId(), containerID);
+ applicationId, containerId);
break;
case CONTAINER_CLEANEDUP_AFTER_KILL:
metrics.endRunningContainer();
@@ -413,21 +427,21 @@ private void finished() {
metrics.killedContainer();
NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
+ applicationId,
+ containerId);
}
- metrics.releaseContainer(this.container.getResource());
+ metrics.releaseContainer(this.resource);
// Inform the application
@SuppressWarnings("rawtypes")
EventHandler eventHandler = dispatcher.getEventHandler();
- eventHandler.handle(new ApplicationContainerFinishedEvent(containerID));
+ eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
// Remove the container from the resource-monitor
- eventHandler.handle(new ContainerStopMonitoringEvent(containerID));
+ eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
// Tell the logService too
eventHandler.handle(new LogHandlerContainerFinishedEvent(
- containerID, exitCode));
+ containerId, exitCode));
}
@SuppressWarnings("unchecked") // dispatcher not typed
@@ -489,7 +503,7 @@ public ContainerState transition(ContainerImpl container,
for (Map.Entry service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
- container.user, container.container.getId()
+ container.user, container.containerId
.getApplicationAttemptId().getApplicationId(),
service.getKey().toString(), service.getValue()));
}
@@ -574,7 +588,7 @@ public ContainerState transition(ContainerImpl container,
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.container.getId());
+ " for container " + container.containerId);
assert false;
// fail container?
return ContainerState.LOCALIZING;
@@ -602,14 +616,14 @@ public void transition(ContainerImpl container, ContainerEvent event) {
// Inform the ContainersMonitor to start monitoring the container's
// resource usage.
long pmemBytes =
- container.container.getResource().getMemory() * 1024 * 1024L;
+ container.getResource().getMemory() * 1024 * 1024L;
float pmemRatio = container.daemonConf.getFloat(
YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
long vmemBytes = (long) (pmemRatio * pmemBytes);
container.dispatcher.getEventHandler().handle(
- new ContainerStartMonitoringEvent(container.container.getId(),
+ new ContainerStartMonitoringEvent(container.containerId,
vmemBytes, pmemBytes));
container.metrics.runningContainer();
}
@@ -743,7 +757,7 @@ public void transition(ContainerImpl container, ContainerEvent event) {
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.container.getId());
+ " for container " + container.containerId);
assert false;
// fail container?
return;
@@ -848,7 +862,7 @@ public void handle(ContainerEvent event) {
public String toString() {
this.readLock.lock();
try {
- return ConverterUtils.toString(container.getId());
+ return ConverterUtils.toString(this.containerId);
} finally {
this.readLock.unlock();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index d5142a1..0b5bc69 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -22,8 +22,8 @@
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import java.io.DataOutputStream;
-import java.io.IOException;
import java.io.File;
+import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -53,12 +53,12 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
@@ -68,7 +68,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable {
@@ -86,6 +85,7 @@
private final Application app;
private final Container container;
private final Configuration conf;
+ private final Context context;
private volatile AtomicBoolean shouldLaunchContainer = new AtomicBoolean(false);
private volatile AtomicBoolean completed = new AtomicBoolean(false);
@@ -97,9 +97,10 @@
private final LocalDirsHandlerService dirsHandler;
- public ContainerLaunch(Configuration configuration, Dispatcher dispatcher,
- ContainerExecutor exec, Application app, Container container,
- LocalDirsHandlerService dirsHandler) {
+ public ContainerLaunch(Context context, Configuration configuration,
+ Dispatcher dispatcher, ContainerExecutor exec, Application app,
+ Container container, LocalDirsHandlerService dirsHandler) {
+ this.context = context;
this.conf = configuration;
this.app = app;
this.exec = exec;
@@ -120,7 +121,7 @@ public Integer call() {
final ContainerLaunchContext launchContext = container.getLaunchContext();
final Map> localResources =
container.getLocalizedResources();
- ContainerId containerID = container.getContainer().getId();
+ ContainerId containerID = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerID);
final List command = launchContext.getCommands();
int ret = -1;
@@ -301,7 +302,7 @@ public Integer call() {
* @throws IOException
*/
public void cleanupContainer() throws IOException {
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerId);
LOG.info("Cleaning up container " + containerIdStr);
@@ -372,7 +373,7 @@ public void cleanupContainer() throws IOException {
*/
private String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
- ConverterUtils.toString(container.getContainer().getId());
+ ConverterUtils.toString(container.getContainerId());
String processId = null;
LOG.debug("Accessing pid for container " + containerIdStr
+ " from pid file " + pidFilePath);
@@ -550,16 +551,16 @@ public void sanitizeEnv(Map environment,
*/
environment.put(Environment.CONTAINER_ID.name(), container
- .getContainer().getId().toString());
+ .getContainerId().toString());
environment.put(Environment.NM_PORT.name(),
- String.valueOf(container.getContainer().getNodeId().getPort()));
+ String.valueOf(this.context.getNodeId().getPort()));
- environment.put(Environment.NM_HOST.name(), container.getContainer()
- .getNodeId().getHost());
+ environment.put(Environment.NM_HOST.name(), this.context.getNodeId()
+ .getHost());
- environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer()
- .getNodeHttpAddress().split(":")[1]);
+ environment.put(Environment.NM_HTTP_PORT.name(),
+ String.valueOf(this.context.getHttpPort()));
environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 22cffb9..0fb963a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -36,8 +36,8 @@
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
@@ -111,15 +111,16 @@ public void stop() {
public void handle(ContainersLauncherEvent event) {
// TODO: ContainersLauncher launches containers one by one!!
Container container = event.getContainer();
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
Application app =
context.getApplications().get(
containerId.getApplicationAttemptId().getApplicationId());
- ContainerLaunch launch = new ContainerLaunch(getConfig(), dispatcher,
- exec, app, event.getContainer(), dirsHandler);
+ ContainerLaunch launch =
+ new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
+ event.getContainer(), dirsHandler);
running.put(containerId,
new RunningContainer(containerLauncher.submit(launch),
launch));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index abd8044..a44a99d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -359,14 +359,14 @@ private void handleInitContainerResources(
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
LocalizerContext ctxt = new LocalizerContext(
- c.getUser(), c.getContainer().getId(), c.getCredentials());
+ c.getUser(), c.getContainerId(), c.getCredentials());
Map> rsrcs =
rsrcReqs.getRequestedResources();
for (Map.Entry> e :
rsrcs.entrySet()) {
LocalResourcesTracker tracker =
getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainer().getId().getApplicationAttemptId()
+ c.getContainerId().getApplicationAttemptId()
.getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
@@ -396,21 +396,21 @@ private void handleCleanupContainerResources(
for (Map.Entry> e :
rsrcs.entrySet()) {
LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainer().getId().getApplicationAttemptId()
+ c.getContainerId().getApplicationAttemptId()
.getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceReleaseEvent(req,
- c.getContainer().getId()));
+ c.getContainerId()));
}
}
- String locId = ConverterUtils.toString(c.getContainer().getId());
+ String locId = ConverterUtils.toString(c.getContainerId());
localizerTracker.cleanupPrivLocalizers(locId);
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
String appIDStr = ConverterUtils.toString(
- c.getContainer().getId().getApplicationAttemptId().getApplicationId());
+ c.getContainerId().getApplicationAttemptId().getApplicationId());
for (String localDir : dirsHandler.getLocalDirs()) {
// Delete the user-owned container-dir
@@ -430,7 +430,7 @@ private void handleCleanupContainerResources(
}
dispatcher.getEventHandler().handle(
- new ContainerEvent(c.getContainer().getId(),
+ new ContainerEvent(c.getContainerId(),
ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
index 3128217..2e74a3b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
@@ -20,7 +20,6 @@
import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -43,6 +42,7 @@
private final Context nmContext;
private final NMWebApp nmWebApp;
private WebApp webApp;
+ private int port;
public WebServer(Context nmContext, ResourceView resourceView,
ApplicationACLsManager aclsManager,
@@ -66,9 +66,7 @@ public synchronized void start() {
this.webApp =
WebApps.$for("node", Context.class, this.nmContext, "ws")
.at(bindAddress).with(getConfig()).start(this.nmWebApp);
- int port = this.webApp.httpServer().getPort();
- String webAddress = StringUtils.split(bindAddress, ':')[0] + ":" + port;
- getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddress);
+ this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";
LOG.error(msg, e);
@@ -77,6 +75,10 @@ public synchronized void start() {
super.start();
}
+ public int getPort() {
+ return this.port;
+ }
+
@Override
public synchronized void stop() {
if (this.webApp != null) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
index 379d998..85d2b82 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
@@ -29,7 +29,6 @@
import org.apache.hadoop.yarn.api.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -60,7 +59,7 @@ public ContainerInfo(final Context nmContext, final Container container) {
public ContainerInfo(final Context nmContext, final Container container,
String requestUri, String pathPrefix) {
- this.id = container.getContainer().getId().toString();
+ this.id = container.getContainerId().toString();
this.nodeId = nmContext.getNodeId().toString();
ContainerStatus containerData = container.cloneAndGetContainerStatus();
this.exitCode = containerData.getExitStatus();
@@ -74,7 +73,7 @@ public ContainerInfo(final Context nmContext, final Container container,
}
this.user = container.getUser();
- Resource res = container.getContainer().getResource();
+ Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 4a2eb38..63b0fd4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -20,7 +20,6 @@
import static org.junit.Assert.fail;
-import java.io.IOException;
import java.util.Collection;
import org.apache.commons.logging.Log;
@@ -54,7 +53,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.util.BuilderUtils;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -94,10 +92,10 @@ public void handle(LocalizationEvent event) {
.getRequestedResources().values()) {
for (LocalResourceRequest req : rc) {
LOG.info("DEBUG: " + req + ":"
- + rsrcReqs.getContainer().getContainer().getId());
+ + rsrcReqs.getContainer().getContainerId());
dispatcher.getEventHandler().handle(
new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
- .getContainer().getId(), req, new Path("file:///local"
+ .getContainerId(), req, new Path("file:///local"
+ req.getPath().toUri().getPath())));
}
}
@@ -107,7 +105,7 @@ public void handle(LocalizationEvent event) {
((ContainerLocalizationEvent) event).getContainer();
// TODO: delete the container dir
this.dispatcher.getEventHandler().handle(
- new ContainerEvent(container.getContainer().getId(),
+ new ContainerEvent(container.getContainerId(),
ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
break;
case DESTROY_APPLICATION_RESOURCES:
@@ -136,7 +134,7 @@ protected ContainersLauncher createContainersLauncher(Context context,
@Override
public void handle(ContainersLauncherEvent event) {
Container container = event.getContainer();
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
dispatcher.getEventHandler().handle(
@@ -183,25 +181,15 @@ public void setBlockNewContainerRequests(boolean blockNewContainerRequests) {
@Override
protected void authorizeRequest(String containerIDStr,
ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
throws YarnRemoteException {
// do Nothing
}
@Override
- protected ContainerTokenIdentifier getContainerTokenIdentifier(
- UserGroupInformation remoteUgi,
- org.apache.hadoop.yarn.api.records.Container container)
- throws YarnRemoteException {
- try {
- return new ContainerTokenIdentifier(container.getId(),
- container.getNodeHttpAddress(), remoteUgi.getUserName(),
- container.getResource(), System.currentTimeMillis() + 100000l, 123,
- BuilderUtils.newContainerTokenIdentifier(
- container.getContainerToken()).getRMIdentifer());
- } catch (IOException e) {
- throw new YarnRemoteException(e);
- }
+ protected ContainerTokenIdentifier
+ getContainerTokenIdentifier(UserGroupInformation remoteUgi,
+ ContainerTokenIdentifier containerTokenId) throws YarnRemoteException {
+ return containerTokenId;
}
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index 928aec7..06830d7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.File;
import java.io.IOException;
@@ -30,7 +27,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -80,7 +76,12 @@ public void testSuccessfulContainerLaunch() throws InterruptedException,
YarnConfiguration conf = new YarnConfiguration();
- Context context = new NMContext(new NMContainerTokenSecretManager(conf));
+ Context context = new NMContext(new NMContainerTokenSecretManager(conf)) {
+ @Override
+ public int getHttpPort() {
+ return 1234;
+ }
+ };
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
@@ -132,10 +133,7 @@ public long getRMIdentifier() {
applicationAttemptId.setApplicationId(applicationId);
applicationAttemptId.setAttemptId(0);
cID.setApplicationAttemptId(applicationAttemptId);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cID);
Resource r = BuilderUtils.newResource(1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
String user = "testing";
String host = "127.0.0.1";
int port = 1234;
@@ -143,11 +141,10 @@ public long getRMIdentifier() {
BuilderUtils.newContainerToken(cID, host, port, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
SIMULATED_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
- request.setContainer(mockContainer);
+ request.setContainerToken(containerToken);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index a96d6ee..f09f5a8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -185,10 +185,7 @@ private int runAndBlock(ContainerId cId, String ... cmd) throws IOException {
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap env = new HashMap();
- org.apache.hadoop.yarn.api.records.Container containerAPI =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(container.getContainer()).thenReturn(containerAPI);
- when(container.getContainer().getId()).thenReturn(cId);
+ when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(context.getEnvironment()).thenReturn(env);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index b26889a..8b59138 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -34,6 +34,7 @@
import java.util.List;
import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -108,10 +109,7 @@ public void testContainerLaunch() throws IOException {
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap env = new HashMap();
- org.apache.hadoop.yarn.api.records.Container containerAPI =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(container.getContainer()).thenReturn(containerAPI);
- when(container.getContainer().getId()).thenReturn(cId);
+ when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(cId.toString()).thenReturn(containerId);
@@ -228,10 +226,7 @@ public void testContainerLaunchError() throws IOException {
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap env = new HashMap();
- org.apache.hadoop.yarn.api.records.Container containerAPI =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(container.getContainer()).thenReturn(containerAPI);
- when(container.getContainer().getId()).thenReturn(cId);
+ when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(cId.toString()).thenReturn(containerId);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index ae19097..d97c664 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -112,9 +112,6 @@ public void testClearLocalDirWhenNodeReboot() throws IOException,
Records.newRecord(ContainerLaunchContext.class);
// Construct the Container-id
ContainerId cId = createContainerId();
- org.apache.hadoop.yarn.api.records.Container mockContainer =
- Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class);
- mockContainer.setId(cId);
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -136,20 +133,16 @@ public void testClearLocalDirWhenNodeReboot() throws IOException,
containerLaunchContext.setCommands(commands);
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(1024);
- mockContainer.setResource(resource);
NodeId nodeId = BuilderUtils.newNodeId("127.0.0.1", 12345);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
user, resource, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), 0);
- mockContainer.setContainerToken(containerToken);
- mockContainer.setNodeHttpAddress("127.0.0.1");
- mockContainer.setNodeId(nodeId);
final StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
final UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(cId.toString());
currentUser.doAs(new PrivilegedExceptionAction() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index 320cf43..a550268 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -31,7 +31,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -43,7 +42,6 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.NMNotYetReadyException;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -281,12 +279,10 @@ public void run() {
try {
while (!isStopped && numContainers < 10) {
ContainerId cId = TestNodeManagerShutdown.createContainerId();
- Container container =
- BuilderUtils.newContainer(cId, null, null, null, null, null);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(container);
+ startRequest.setContainerToken(null);
System.out.println("no. of containers to be launched: "
+ numContainers);
numContainers++;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
index 78fc3cb..defb5fa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
@@ -44,7 +44,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -56,7 +55,6 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -158,13 +156,9 @@ public static void startContainer(NodeManager nm, ContainerId cId,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- Container mockContainer = new ContainerPBImpl();
- mockContainer.setId(cId);
NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
- mockContainer.setNodeId(nodeId);
- mockContainer.setNodeHttpAddress("localhost:12345");
-
+
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -183,16 +177,14 @@ public static void startContainer(NodeManager nm, ContainerId cId,
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource resource = BuilderUtils.newResource(1024, 1);
- mockContainer.setResource(resource);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
user, resource, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), 0);
- mockContainer.setContainerToken(containerToken);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(cId.toString());
@@ -285,4 +277,4 @@ public void setMasterKey(MasterKey masterKey) {
getNMContext().getContainerTokenSecretManager().setMasterKey(masterKey);
}
}
-}
\ No newline at end of file
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 7e1d2b6..777d92f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -58,6 +58,7 @@
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
@@ -187,8 +188,6 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
nodeStatus.setResponseId(heartBeatID++);
Map> appToContainers =
getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
- org.apache.hadoop.yarn.api.records.Container mockContainer =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
ApplicationId appId1 = ApplicationId.newInstance(0, 1);
ApplicationId appId2 = ApplicationId.newInstance(0, 2);
@@ -202,12 +201,17 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
firstContainerID.setId(heartBeatID);
ContainerLaunchContext launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- when(mockContainer.getId()).thenReturn(firstContainerID);
Resource resource = BuilderUtils.newResource(2, 1);
- when(mockContainer.getResource()).thenReturn(resource);
+ long currentTime = System.currentTimeMillis();
+ String user = "testUser";
+ ContainerTokenIdentifier containerToken =
+ BuilderUtils.newContainerTokenIdentifier(BuilderUtils
+ .newContainerToken(firstContainerID, "127.0.0.1", 1234, user,
+ resource, currentTime + 10000, 123, "password".getBytes(),
+ currentTime));
Container container =
- new ContainerImpl(conf, mockDispatcher, launchContext,
- mockContainer, null, mockMetrics, null);
+ new ContainerImpl(conf, mockDispatcher, launchContext, null,
+ mockMetrics, containerToken);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -227,12 +231,17 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
secondContainerID.setId(heartBeatID);
ContainerLaunchContext launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- when(mockContainer.getId()).thenReturn(secondContainerID);
+ long currentTime = System.currentTimeMillis();
+ String user = "testUser";
Resource resource = BuilderUtils.newResource(3, 1);
- when(mockContainer.getResource()).thenReturn(resource);
+ ContainerTokenIdentifier containerToken =
+ BuilderUtils.newContainerTokenIdentifier(BuilderUtils
+ .newContainerToken(secondContainerID, "127.0.0.1", 1234, user,
+ resource, currentTime + 10000, 123,
+ "password".getBytes(), currentTime));
Container container =
- new ContainerImpl(conf, mockDispatcher, launchContext,
- mockContainer, null, mockMetrics, null);
+ new ContainerImpl(conf, mockDispatcher, launchContext, null,
+ mockMetrics, containerToken);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 2e60784..1dd8cc2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -33,7 +33,6 @@
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -94,9 +93,14 @@ public BaseContainerManagerTest() throws UnsupportedFileSystemException {
protected static Log LOG = LogFactory
.getLog(BaseContainerManagerTest.class);
+ protected static final int HTTP_PORT = 5412;
protected Configuration conf = new YarnConfiguration();
protected Context context = new NMContext(new NMContainerTokenSecretManager(
- conf));
+ conf)) {
+ public int getHttpPort() {
+ return HTTP_PORT;
+ };
+ };
protected ContainerExecutor exec;
protected DeletionService delSrvc;
protected String user = "nobody";
@@ -177,9 +181,8 @@ public void setup() throws IOException {
@Override
protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext, Container container,
- UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
- throws YarnRemoteException {
+ ContainerLaunchContext launchContext, UserGroupInformation remoteUgi,
+ ContainerTokenIdentifier tokenId) throws YarnRemoteException {
// do nothing
}
};
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 982ba6f..90bd470 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@ -43,7 +40,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -142,24 +138,16 @@ public void testContainerSetup() throws IOException, InterruptedException,
new HashMap();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cId);
Resource r = BuilderUtils.newResource(512, 1);
- when(mockContainer.getResource()).thenReturn(r);
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
@@ -243,23 +231,16 @@ public void testContainerLaunchAndStop() throws IOException,
containerLaunchContext.setLocalResources(localResources);
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cId);
Resource r = BuilderUtils.newResource(100, 1);
- when(mockContainer.getResource()).thenReturn(r); // MB
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -358,22 +339,15 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException,
containerLaunchContext.setLocalResources(localResources);
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cId);
Resource r = BuilderUtils.newResource(100, 1);
- when(mockContainer.getResource()).thenReturn(r); // MB
int port = 12345;
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -453,24 +427,16 @@ public void testLocalFilesCleanup() throws InterruptedException,
new HashMap();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cId);
Resource r = BuilderUtils.newResource(100, 1);
- when(mockContainer.getResource()).thenReturn(r);
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
-// containerLaunchContext.command = new ArrayList();
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
- request.setContainer(mockContainer);
+ request.setContainerToken(containerToken);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -549,8 +515,6 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
.setLocalResources(new HashMap());
Resource mockResource = BuilderUtils.newResource(1024, 1);
- Container mockContainer1 = mock(Container.class);
- when(mockContainer1.getId()).thenReturn(cId1);
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
@@ -560,8 +524,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes(),
(long) ResourceManagerConstants.RM_INVALID_IDENTIFIER);
- when(mockContainer1.getContainerToken()).thenReturn(containerToken1);
- startRequest1.setContainer(mockContainer1);
+ startRequest1.setContainerToken(containerToken1);
boolean catchException = false;
try {
containerManager.startContainer(startRequest1);
@@ -579,10 +542,6 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
Assert.assertTrue(catchException);
// Construct the Container with a RMIdentifier within current RM
- Container mockContainer2 = mock(Container.class);
- when(mockContainer2.getId()).thenReturn(cId2);
-
- when(mockContainer2.getResource()).thenReturn(mockResource);
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
@@ -590,9 +549,7 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
System.currentTimeMillis() + 10000, 123, "password".getBytes(),
super.DUMMY_RM_IDENTIFIER);
- when(mockContainer2.getContainerToken()).thenReturn(containerToken2);
-
- startRequest2.setContainer(mockContainer2);
+ startRequest2.setContainerToken(containerToken2);
boolean noException = true;
try {
containerManager.startContainer(startRequest2);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index f29bcd9..6081b9a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -91,7 +91,7 @@ public void testApplicationInit1() {
for (int i = 0; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(i)
- .getContainer().getId())));
+ .getContainerId())));
}
} finally {
if (wa != null)
@@ -116,7 +116,7 @@ public void testApplicationInit2() {
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(0)
- .getContainer().getId())));
+ .getContainerId())));
wa.initContainer(1);
wa.initContainer(2);
@@ -126,7 +126,7 @@ public void testApplicationInit2() {
for (int i = 1; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(i)
- .getContainer().getId())));
+ .getContainerId())));
}
} finally {
if (wa != null)
@@ -241,7 +241,7 @@ public void testAppFinishedOnRunningContainers() {
for (int i = 1; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerKillMatcher(wa.containers.get(i)
- .getContainer().getId())));
+ .getContainerId())));
}
wa.containerFinished(1);
@@ -267,7 +267,7 @@ public void testAppFinishedOnRunningContainers() {
wa.appResourcesCleanedup();
for ( Container container : wa.containers) {
Assert.assertTrue(wa.context.getContainerTokenSecretManager()
- .isValidStartContainerRequest(container.getContainer().getId()));
+ .isValidStartContainerRequest(container.getContainerId()));
}
assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
@@ -307,7 +307,7 @@ public void testAppFinishedOnCompletedContainers() {
wa.appResourcesCleanedup();
for ( Container container : wa.containers) {
Assert.assertTrue(wa.context.getContainerTokenSecretManager()
- .isValidStartContainerRequest(container.getContainer().getId()));
+ .isValidStartContainerRequest(container.getContainerId()));
}
assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
} finally {
@@ -370,7 +370,7 @@ public void testAppFinishedOnIniting() {
verify(wa.containerBus).handle(
argThat(new ContainerKillMatcher(wa.containers.get(0)
- .getContainer().getId())));
+ .getContainerId())));
assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT,
wa.app.getApplicationState());
@@ -487,10 +487,10 @@ public boolean matches(Object argument) {
containers.add(container);
long currentTime = System.currentTimeMillis();
context.getContainerTokenSecretManager().startContainerSuccessful(
- new ContainerTokenIdentifier(container.getContainer().getId(), "",
+ new ContainerTokenIdentifier(container.getContainerId(), "",
"", null, currentTime + 1000, masterKey.getKeyId(), currentTime));
Assert.assertFalse(context.getContainerTokenSecretManager()
- .isValidStartContainerRequest(container.getContainer().getId()));
+ .isValidStartContainerRequest(container.getContainerId()));
}
dispatcher.start();
@@ -522,7 +522,7 @@ public void initContainer(int containerNum) {
public void containerFinished(int containerNum) {
app.handle(new ApplicationContainerFinishedEvent(containers.get(
- containerNum).getContainer().getId()));
+ containerNum).getContainerId()));
drainDispatcherEvents();
}
@@ -549,10 +549,7 @@ private Container createMockedContainer(ApplicationId appId, int containerId) {
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
Container c = mock(Container.class);
- org.apache.hadoop.yarn.api.records.Container containerAPI =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(c.getContainer()).thenReturn(containerAPI);
- when(c.getContainer().getId()).thenReturn(cId);
+ when(c.getContainerId()).thenReturn(cId);
ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
when(c.getLaunchContext()).thenReturn(launchContext);
when(launchContext.getApplicationACLs()).thenReturn(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 946461a..5e4662d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -52,7 +52,6 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -377,7 +376,7 @@ public boolean matches(Object o) {
public boolean matches(Object o) {
ContainersLauncherEvent evt = (ContainersLauncherEvent) o;
return evt.getType() == ContainersLauncherEventType.LAUNCH_CONTAINER
- && wcf.cId == evt.getContainer().getContainer().getId();
+ && wcf.cId == evt.getContainer().getContainerId();
}
};
verify(wc.launcherBus).handle(argThat(matchesLaunchReq));
@@ -526,13 +525,6 @@ public boolean matches(Object o) {
return serviceData;
}
- private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
- org.apache.hadoop.yarn.api.records.Container container,
- ContainerTokenIdentifier identifier) throws IOException {
- return new ContainerImpl(conf, disp, ctx, container, null, metrics,
- identifier);
- }
-
@SuppressWarnings("unchecked")
private class WrappedContainer {
final DrainDispatcher dispatcher;
@@ -612,7 +604,7 @@ private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
}
when(ctxt.getServiceData()).thenReturn(serviceData);
- c = newContainer(dispatcher, ctxt, mockContainer, identifier);
+ c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier);
dispatcher.start();
}
@@ -649,7 +641,7 @@ public void initContainer() {
Path p = new Path(cache, rsrc.getKey());
localPaths.put(p, Arrays.asList(rsrc.getKey()));
// rsrc copied to p
- c.handle(new ContainerResourceLocalizedEvent(c.getContainer().getId(),
+ c.handle(new ContainerResourceLocalizedEvent(c.getContainerId(),
req, p));
}
drainDispatcherEvents();
@@ -672,8 +664,8 @@ public void failLocalizeSpecificResource(String rsrcKey)
LocalResource rsrc = localResources.get(rsrcKey);
LocalResourceRequest req = new LocalResourceRequest(rsrc);
Exception e = new Exception("Fake localization error");
- c.handle(new ContainerResourceFailedEvent(c.getContainer()
- .getId(), req, e.getMessage()));
+ c.handle(new ContainerResourceFailedEvent(c.getContainerId(), req, e
+ .getMessage()));
drainDispatcherEvents();
}
@@ -688,7 +680,7 @@ public void failLocalizeResources(int failRsrcCount)
++counter;
LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
Exception e = new Exception("Fake localization error");
- c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(),
+ c.handle(new ContainerResourceFailedEvent(c.getContainerId(),
req, e.getMessage()));
}
drainDispatcherEvents();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 0252eab..ff4e493 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File;
@@ -47,7 +45,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -165,7 +162,6 @@ public void testContainerEnvVariables() throws Exception {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
@@ -176,11 +172,6 @@ public void testContainerEnvVariables() throws Exception {
recordFactory.newRecordInstance(ContainerId.class);
int port = 12345;
cId.setApplicationAttemptId(appAttemptId);
- when(mockContainer.getId()).thenReturn(cId);
-
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
Map userSetEnv = new HashMap();
userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
@@ -244,15 +235,13 @@ public void testContainerEnvVariables() throws Exception {
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 1234,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -275,27 +264,20 @@ public void testContainerEnvVariables() throws Exception {
BufferedReader reader =
new BufferedReader(new FileReader(processStartFile));
Assert.assertEquals(cId.toString(), reader.readLine());
- Assert.assertEquals(mockContainer.getNodeId().getHost(),
- reader.readLine());
- Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()),
- reader.readLine());
- Assert.assertEquals(
- String.valueOf(mockContainer.getNodeHttpAddress().split(":")[1]),
- reader.readLine());
+ Assert.assertEquals(context.getNodeId().getHost(), reader.readLine());
+ Assert.assertEquals(String.valueOf(context.getNodeId().getPort()),
+ reader.readLine());
+ Assert.assertEquals(String.valueOf(HTTP_PORT), reader.readLine());
Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine());
Assert.assertEquals(cId.toString(), containerLaunchContext
.getEnvironment().get(Environment.CONTAINER_ID.name()));
- Assert.assertEquals(mockContainer.getNodeId().getHost(),
- containerLaunchContext.getEnvironment()
- .get(Environment.NM_HOST.name()));
- Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()),
- containerLaunchContext.getEnvironment().get(
- Environment.NM_PORT.name()));
- Assert.assertEquals(
- mockContainer.getNodeHttpAddress().split(":")[1],
- containerLaunchContext.getEnvironment().get(
- Environment.NM_HTTP_PORT.name()));
+ Assert.assertEquals(context.getNodeId().getHost(), containerLaunchContext
+ .getEnvironment().get(Environment.NM_HOST.name()));
+ Assert.assertEquals(String.valueOf(context.getNodeId().getPort()),
+ containerLaunchContext.getEnvironment().get(Environment.NM_PORT.name()));
+ Assert.assertEquals(String.valueOf(HTTP_PORT), containerLaunchContext
+ .getEnvironment().get(Environment.NM_HTTP_PORT.name()));
Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext
.getEnvironment().get(Environment.LOCAL_DIRS.name()));
// Get the pid of the process
@@ -337,7 +319,6 @@ public void testContainerEnvVariables() throws Exception {
public void testDelayedKill() throws Exception {
containerManager.start();
- Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationAttemptId appAttemptId =
@@ -374,11 +355,7 @@ public void testDelayedKill() throws Exception {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- when(mockContainer.getId()).thenReturn(cId);
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
// upload the script file so that the container can run it
URL resource_alpha =
@@ -401,15 +378,13 @@ public void testDelayedKill() throws Exception {
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
ContainerToken containerToken =
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index cd12ea5..3626626 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -495,7 +495,7 @@ public boolean matches(Object o) {
Thread.sleep(1000);
dispatcher.await();
String appStr = ConverterUtils.toString(appId);
- String ctnrStr = c.getContainer().getId().toString();
+ String ctnrStr = c.getContainerId().toString();
ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class);
verify(exec).startLocalizer(tokenPathCaptor.capture(),
isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr),
@@ -571,7 +571,7 @@ public boolean matches(Object o) {
public boolean matches(Object o) {
ContainerEvent evt = (ContainerEvent) o;
return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED
- && c.getContainer().getId() == evt.getContainerID();
+ && c.getContainerId() == evt.getContainerID();
}
};
// total 2 resource localzation calls. one for each resource.
@@ -760,11 +760,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception {
// Container - 1
ContainerImpl container1 = createMockContainer(user, 1);
- String localizerId1 = container1.getContainer().getId().toString();
+ String localizerId1 = container1.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId1,
rls.new LocalizerRunner(new LocalizerContext(user, container1
- .getContainer().getId(), null), localizerId1));
+ .getContainerId(), null), localizerId1));
LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1);
dispatcher1.getEventHandler().handle(
@@ -775,11 +775,11 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception {
// Container - 2 now makes the request.
ContainerImpl container2 = createMockContainer(user, 2);
- String localizerId2 = container2.getContainer().getId().toString();
+ String localizerId2 = container2.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId2,
rls.new LocalizerRunner(new LocalizerContext(user, container2
- .getContainer().getId(), null), localizerId2));
+ .getContainerId(), null), localizerId2));
LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2);
dispatcher1.getEventHandler().handle(
createContainerLocalizationEvent(container2,
@@ -920,11 +920,11 @@ public void testLocalResourcePath() throws Exception {
// Container - 1
Container container1 = createMockContainer(user, 1);
- String localizerId1 = container1.getContainer().getId().toString();
+ String localizerId1 = container1.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId1,
rls.new LocalizerRunner(new LocalizerContext(user, container1
- .getContainer().getId(), null), localizerId1));
+ .getContainerId(), null), localizerId1));
// Creating two requests for container
// 1) Private resource
@@ -1317,10 +1317,7 @@ private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
private ContainerImpl createMockContainer(String user, int containerId) {
ContainerImpl container = mock(ContainerImpl.class);
- org.apache.hadoop.yarn.api.records.Container c =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(container.getContainer()).thenReturn(c);
- when(container.getContainer().getId()).thenReturn(
+ when(container.getContainerId()).thenReturn(
BuilderUtils.newContainerId(1, 1, 1, containerId));
when(container.getUser()).thenReturn(user);
Credentials mockCredentials = mock(Credentials.class);
@@ -1360,11 +1357,8 @@ private static Container getMockContainer(ApplicationId appId, int id) {
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id);
- org.apache.hadoop.yarn.api.records.Container containerAPI =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(c.getContainer()).thenReturn(containerAPI);
when(c.getUser()).thenReturn("user0");
- when(c.getContainer().getId()).thenReturn(cId);
+ when(c.getContainerId()).thenReturn(cId);
Credentials creds = new Credentials();
creds.addToken(new Text("tok" + id), getToken(id));
when(c.getCredentials()).thenReturn(creds);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 2a09bb6..05bc408 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -55,10 +55,10 @@
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -707,15 +707,12 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
- when(mockContainer.getId()).thenReturn(cId);
-
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -736,15 +733,14 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
- when(mockContainer.getContainerToken()).thenReturn(
- BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
- System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
- super.DUMMY_RM_IDENTIFIER));
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
+ System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
+ super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
this.containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(this.containerManager,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index ba421b7..27539d3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -22,7 +22,6 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File;
@@ -45,7 +44,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -202,7 +200,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
@@ -212,12 +209,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
cId.setId(0);
cId.setApplicationAttemptId(appAttemptId);
- when(mockContainer.getId()).thenReturn(cId);
-
- when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
int port = 12345;
- when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":" + port);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -239,7 +231,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@@ -247,8 +238,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
port, user, r, System.currentTimeMillis() + 10000L, 123,
"password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
- startRequest.setContainer(mockContainer);
+ startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 13949b0..5dce308 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
@@ -34,10 +31,11 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
@@ -52,7 +50,7 @@
private final Map> resource =
new HashMap>();
private RecordFactory recordFactory;
- private org.apache.hadoop.yarn.api.records.Container mockContainer;
+ private final ContainerTokenIdentifier containerTokenIdentifier;
public MockContainer(ApplicationAttemptId appAttemptId,
Dispatcher dispatcher, Configuration conf, String user,
@@ -65,15 +63,12 @@ public MockContainer(ApplicationAttemptId appAttemptId,
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
long currentTime = System.currentTimeMillis();
- ContainerToken containerToken =
- BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user,
- BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
- "password".getBytes(), currentTime);
+ this.containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(BuilderUtils
+ .newContainerToken(id, "127.0.0.1", 1234, user,
+ BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
+ "password".getBytes(), currentTime));
this.state = ContainerState.NEW;
-
- mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
- when(mockContainer.getId()).thenReturn(id);
}
public void setState(ContainerState state) {
@@ -126,7 +121,17 @@ public void handle(ContainerEvent event) {
}
@Override
- public org.apache.hadoop.yarn.api.records.Container getContainer() {
- return this.mockContainer;
+ public ContainerId getContainerId() {
+ return this.id;
+ }
+
+ @Override
+ public Resource getResource() {
+ return this.containerTokenIdentifier.getResource();
+ }
+
+ @Override
+ public ContainerTokenIdentifier getContainerTokenIdentifier() {
+ return this.containerTokenIdentifier;
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
index dbb6866..cab676a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
@@ -26,7 +26,6 @@
import java.io.IOException;
import java.io.Writer;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -76,7 +75,7 @@ public void tearDown() {
FileUtil.fullyDelete(testLogDir);
}
- private String startNMWebAppServer(String webAddr) {
+ private int startNMWebAppServer(String webAddr) {
Context nmContext = new NodeManager.NMContext(null);
ResourceView resourceView = new ResourceView() {
@Override
@@ -107,20 +106,19 @@ public boolean isPmemCheckEnabled() {
new ApplicationACLsManager(conf), dirsHandler);
server.init(conf);
server.start();
- String webAppAddr = conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS);
- return StringUtils.split(webAppAddr, ':')[1];
+ return server.getPort();
}
@Test
public void testNMWebAppWithOutPort() throws IOException {
- String port = startNMWebAppServer("0.0.0.0");
- Assert.assertTrue("Port is not updated", Integer.parseInt(port) > 0);
+ int port = startNMWebAppServer("0.0.0.0");
+ Assert.assertTrue("Port is not updated", port > 0);
}
@Test
public void testNMWebAppWithEphemeralPort() throws IOException {
- String port = startNMWebAppServer("0.0.0.0:0");
- Assert.assertTrue("Port is not updated", Integer.parseInt(port) > 0);
+ int port = startNMWebAppServer("0.0.0.0:0");
+ Assert.assertTrue("Port is not updated", port > 0);
}
@Test
@@ -180,17 +178,13 @@ public boolean isPmemCheckEnabled() {
// TODO: Use builder utils
ContainerLaunchContext launchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- org.apache.hadoop.yarn.api.records.Container mockContainer =
- mock(org.apache.hadoop.yarn.api.records.Container.class);
long currentTime = System.currentTimeMillis();
ContainerToken containerToken =
BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
"password".getBytes(), currentTime);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
- when(mockContainer.getId()).thenReturn(containerId);
Container container =
- new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
+ new ContainerImpl(conf, dispatcher, launchContext,
null, metrics,
BuilderUtils.newContainerTokenIdentifier(containerToken)) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
index 5333a61..9b059bb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
@@ -188,17 +188,17 @@ public void testNodeAppsNone() throws JSONException, Exception {
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 2);
nmContext.getContainers()
- .put(container1.getContainer().getId(), container1);
+ .put(container1.getContainerId(), container1);
nmContext.getContainers()
- .put(container2.getContainer().getId(), container2);
+ .put(container2.getContainerId(), container2);
- app.getContainers().put(container1.getContainer().getId(), container1);
- app.getContainers().put(container2.getContainer().getId(), container2);
+ app.getContainers().put(container1.getContainerId(), container1);
+ app.getContainers().put(container2.getContainerId(), container2);
HashMap hash = new HashMap();
- hash.put(container1.getContainer().getId().toString(), container1
- .getContainer().getId().toString());
- hash.put(container2.getContainer().getId().toString(), container2
- .getContainer().getId().toString());
+ hash.put(container1.getContainerId().toString(), container1
+ .getContainerId().toString());
+ hash.put(container2.getContainerId().toString(), container2
+ .getContainerId().toString());
return hash;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 5112c6c..b9964ba 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -92,9 +93,15 @@
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext(null);
- nmContext.getNodeId().setHost("testhost.foo.com");
- nmContext.getNodeId().setPort(8042);
+ nmContext = new NodeManager.NMContext(null) {
+ public NodeId getNodeId() {
+ return NodeId.newInstance("testhost.foo.com", 8042);
+ };
+
+ public int getHttpPort() {
+ return 1234;
+ };
+ };
resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -189,17 +196,17 @@ public void testNodeContainersNone() throws JSONException, Exception {
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 2);
nmContext.getContainers()
- .put(container1.getContainer().getId(), container1);
+ .put(container1.getContainerId(), container1);
nmContext.getContainers()
- .put(container2.getContainer().getId(), container2);
+ .put(container2.getContainerId(), container2);
- app.getContainers().put(container1.getContainer().getId(), container1);
- app.getContainers().put(container2.getContainer().getId(), container2);
+ app.getContainers().put(container1.getContainerId(), container1);
+ app.getContainers().put(container2.getContainerId(), container2);
HashMap hash = new HashMap();
- hash.put(container1.getContainer().getId().toString(), container1
- .getContainer().getId().toString());
- hash.put(container2.getContainer().getId().toString(), container2
- .getContainer().getId().toString());
+ hash.put(container1.getContainerId().toString(), container1
+ .getContainerId().toString());
+ hash.put(container2.getContainerId().toString(), container2
+ .getContainerId().toString());
return hash;
}
@@ -472,7 +479,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id,
String state, String user, int exitCode, String diagnostics,
String nodeId, int totalMemoryNeededMB, String logsLink)
throws JSONException, Exception {
- WebServicesTestUtils.checkStringMatch("id", cont.getContainer().getId()
+ WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
.toString(), id);
WebServicesTestUtils.checkStringMatch("state", cont.getContainerState()
.toString(), state);
@@ -484,9 +491,11 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id,
WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId()
.toString(), nodeId);
- assertEquals("totalMemoryNeededMB wrong", 0, totalMemoryNeededMB);
+ assertEquals("totalMemoryNeededMB wrong",
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ totalMemoryNeededMB);
String shortLink =
- ujoin("containerlogs", cont.getContainer().getId().toString(),
+ ujoin("containerlogs", cont.getContainerId().toString(),
cont.getUser());
assertTrue("containerLogsLink wrong", logsLink.contains(shortLink));
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 7c8f1d0..b95d2aa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -109,7 +109,7 @@ private void launch() throws IOException, YarnRemoteException {
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
- request.setContainer(masterContainer);
+ request.setContainerToken(masterContainer.getContainerToken());
containerMgrProxy.startContainer(request);
LOG.info("Done launching container " + masterContainer
+ " for AM " + application.getAppAttemptId());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
index 841bc28..2c70c0e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@@ -342,7 +342,7 @@ private synchronized void assign(Priority priority, NodeType type,
// Launch the container
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(createCLC());
- startRequest.setContainer(container);
+ startRequest.setContainerToken(container.getContainerToken());
nodeManager.startContainer(startRequest);
break;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index 6018a03..24261c2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -42,6 +42,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -49,6 +50,7 @@
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
@@ -161,11 +163,19 @@ public void heartbeat() throws IOException, YarnRemoteException {
synchronized public StartContainerResponse startContainer(
StartContainerRequest request)
throws YarnRemoteException {
- Container requestContainer = request.getContainer();
- ApplicationId applicationId =
- requestContainer.getId().getApplicationAttemptId().
- getApplicationId();
+ ContainerToken containerToken = request.getContainerToken();
+ ContainerTokenIdentifier tokenId = null;
+
+ try {
+ tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+
+ ContainerId containerID = tokenId.getContainerID();
+ ApplicationId applicationId =
+ containerID.getApplicationAttemptId().getApplicationId();
List applicationContainers = containers.get(applicationId);
if (applicationContainers == null) {
@@ -175,18 +185,18 @@ synchronized public StartContainerResponse startContainer(
// Sanity check
for (Container container : applicationContainers) {
- if (container.getId().compareTo(requestContainer.getId())
+ if (container.getId().compareTo(containerID)
== 0) {
throw new IllegalStateException(
- "Container " + requestContainer.getId() +
+ "Container " + containerID +
" already setup on node " + containerManagerAddress);
}
}
Container container =
- BuilderUtils.newContainer(requestContainer.getId(),
+ BuilderUtils.newContainer(containerID,
this.nodeId, nodeHttpAddress,
- requestContainer.getResource(),
+ tokenId.getResource(),
null, null // DKDC - Doesn't matter
);
@@ -195,8 +205,8 @@ synchronized public StartContainerResponse startContainer(
"", -1000);
applicationContainers.add(container);
containerStatusMap.put(container, containerStatus);
- Resources.subtractFrom(available, requestContainer.getResource());
- Resources.addTo(used, requestContainer.getResource());
+ Resources.subtractFrom(available, tokenId.getResource());
+ Resources.addTo(used, tokenId.getResource());
if(LOG.isDebugEnabled()) {
LOG.debug("startContainer:" + " node=" + containerManagerAddress
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index c7679c8..33261ad 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.io.IOException;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -33,11 +34,15 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -57,8 +62,6 @@
String attemptIdAtContainerManager = null;
String containerIdAtContainerManager = null;
String nmHostAtContainerManager = null;
- int nmPortAtContainerManager;
- int nmHttpPortAtContainerManager;
long submitTimeAtContainerManager;
int maxAppAttempts;
@@ -70,17 +73,21 @@
launched = true;
Map env =
request.getContainerLaunchContext().getEnvironment();
- ContainerId containerId =
- request.getContainer().getId();
+
+ ContainerToken containerToken = request.getContainerToken();
+ ContainerTokenIdentifier tokenId = null;
+
+ try {
+ tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+
+ ContainerId containerId = tokenId.getContainerID();
containerIdAtContainerManager = containerId.toString();
attemptIdAtContainerManager =
containerId.getApplicationAttemptId().toString();
- nmHostAtContainerManager = request.getContainer().getNodeId().getHost();
- nmPortAtContainerManager =
- request.getContainer().getNodeId().getPort();
- nmHttpPortAtContainerManager =
- Integer.parseInt(request.getContainer().getNodeHttpAddress()
- .split(":")[1]);
+ nmHostAtContainerManager = tokenId.getNmHostAddress();
submitTimeAtContainerManager =
Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
maxAppAttempts =
@@ -135,12 +142,8 @@ public void testAMLaunchAndCleanup() throws Exception {
Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
.getMasterContainer().getId()
.toString(), containerManager.containerIdAtContainerManager);
- Assert.assertEquals(nm1.getNodeId().getHost(),
- containerManager.nmHostAtContainerManager);
- Assert.assertEquals(nm1.getNodeId().getPort(),
- containerManager.nmPortAtContainerManager);
- Assert.assertEquals(nm1.getHttpPort(),
- containerManager.nmHttpPortAtContainerManager);
+ Assert.assertEquals(nm1.getNodeId().toString(),
+ containerManager.nmHostAtContainerManager);
Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
containerManager.maxAppAttempts);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 70c559f..b1f0ca2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -38,9 +38,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,7 +47,6 @@
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
@@ -121,8 +118,8 @@ private void testContainerManager() throws Exception {
// Testing for malicious user
testMaliceUser();
- // Testing for unauthorized user
- testUnauthorizedUser();
+ // Testing for usage of expired tokens
+ testExpiredTokens();
} finally {
if (yarnCluster != null) {
@@ -184,6 +181,15 @@ public Void run() throws Exception {
resourceManager.getClientRMService().forceKillApplication(request);
}
+ /**
+ * This tests a malice user getting a proper token but then messing with it by
+ * tampering with containerID/Resource etc.. His/her containers should be
+ * rejected.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws YarnRemoteException
+ */
private void testMaliceUser() throws IOException, InterruptedException,
YarnRemoteException {
@@ -205,30 +211,60 @@ private void testMaliceUser() throws IOException, InterruptedException,
appID);
// Now talk to the NM for launching the container with modified resource
- final ContainerId containerID = allocatedContainer.getId();
- UserGroupInformation maliceUser = UserGroupInformation
- .createRemoteUser(containerID.toString());
ContainerToken containerToken = allocatedContainer.getContainerToken();
- byte[] identifierBytes = containerToken.getIdentifier().array();
-
- DataInputBuffer di = new DataInputBuffer();
- di.reset(identifierBytes, identifierBytes.length);
-
- ContainerTokenIdentifier dummyIdentifier = new ContainerTokenIdentifier();
- dummyIdentifier.readFields(di);
+ ContainerTokenIdentifier originalContainerTokenId =
+ BuilderUtils.newContainerTokenIdentifier(containerToken);
// Malice user modifies the resource amount
Resource modifiedResource = BuilderUtils.newResource(2048, 1);
ContainerTokenIdentifier modifiedIdentifier =
- new ContainerTokenIdentifier(dummyIdentifier.getContainerID(),
- dummyIdentifier.getNmHostAddress(), "testUser", modifiedResource,
- Long.MAX_VALUE, dummyIdentifier.getMasterKeyId(),
+ new ContainerTokenIdentifier(originalContainerTokenId.getContainerID(),
+ originalContainerTokenId.getNmHostAddress(), "testUser",
+ modifiedResource, Long.MAX_VALUE,
+ originalContainerTokenId.getMasterKeyId(),
ResourceManager.clusterTimeStamp);
- Token modifiedToken = new Token(
- modifiedIdentifier.getBytes(), containerToken.getPassword().array(),
- new Text(containerToken.getKind()), new Text(containerToken
- .getService()));
+ Token modifiedToken =
+ new Token(modifiedIdentifier.getBytes(),
+ containerToken.getPassword().array(), new Text(
+ containerToken.getKind()), new Text(containerToken.getService()));
+ makeTamperedStartContainerCall(yarnRPC, allocatedContainer,
+ modifiedIdentifier, modifiedToken);
+
+ // Malice user modifies the container-Id
+ ContainerId newContainerId =
+ BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(originalContainerTokenId
+ .getContainerID().getApplicationAttemptId().getApplicationId(), 1),
+ originalContainerTokenId.getContainerID().getId() + 42);
+ modifiedIdentifier =
+ new ContainerTokenIdentifier(newContainerId,
+ originalContainerTokenId.getNmHostAddress(), "testUser",
+ originalContainerTokenId.getResource(), Long.MAX_VALUE,
+ originalContainerTokenId.getMasterKeyId(),
+ ResourceManager.clusterTimeStamp);
+ modifiedToken =
+ new Token(modifiedIdentifier.getBytes(),
+ containerToken.getPassword().array(), new Text(
+ containerToken.getKind()), new Text(containerToken.getService()));
+ makeTamperedStartContainerCall(yarnRPC, allocatedContainer,
+ modifiedIdentifier, modifiedToken);
+
+ // Similarly messing with anything else will fail.
+
+ KillApplicationRequest request = Records
+ .newRecord(KillApplicationRequest.class);
+ request.setApplicationId(appID);
+ resourceManager.getClientRMService().forceKillApplication(request);
+ }
+
+ private void makeTamperedStartContainerCall(final YarnRPC yarnRPC,
+ final Container allocatedContainer,
+ final ContainerTokenIdentifier modifiedIdentifier,
+ Token modifiedToken) {
+ final ContainerId containerID = allocatedContainer.getId();
+ UserGroupInformation maliceUser = UserGroupInformation
+ .createRemoteUser(containerID.toString());
maliceUser.addToken(modifiedToken);
maliceUser.doAs(new PrivilegedAction() {
@Override
@@ -239,11 +275,14 @@ public Void run() {
conf);
LOG.info("Going to contact NM: ilLegal request");
- GetContainerStatusRequest request = recordFactory
- .newRecordInstance(GetContainerStatusRequest.class);
- request.setContainerId(containerID);
+ StartContainerRequest request =
+ Records.newRecord(StartContainerRequest.class);
try {
- client.getContainerStatus(request);
+ request.setContainerToken(allocatedContainer.getContainerToken());
+ ContainerLaunchContext context =
+ createContainerLaunchContextForTest(modifiedIdentifier);
+ request.setContainerLaunchContext(context);
+ client.startContainer(request);
fail("Connection initiation with illegally modified "
+ "tokens is expected to fail.");
} catch (YarnRemoteException e) {
@@ -263,14 +302,9 @@ public Void run() {
return null;
}
});
-
- KillApplicationRequest request = Records
- .newRecord(KillApplicationRequest.class);
- request.setApplicationId(appID);
- resourceManager.getClientRMService().forceKillApplication(request);
}
- private void testUnauthorizedUser() throws IOException, InterruptedException,
+ private void testExpiredTokens() throws IOException, InterruptedException,
YarnRemoteException {
LOG.info("\n\nRunning test for malice user");
@@ -293,48 +327,12 @@ private void testUnauthorizedUser() throws IOException, InterruptedException,
// Now talk to the NM for launching the container with modified containerID
final ContainerId containerID = allocatedContainer.getId();
- /////////// Test calls with illegal containerIDs and illegal Resources
- UserGroupInformation unauthorizedUser = UserGroupInformation
- .createRemoteUser(containerID.toString());
ContainerToken containerToken = allocatedContainer.getContainerToken();
-
- byte[] identifierBytes = containerToken.getIdentifier().array();
- DataInputBuffer di = new DataInputBuffer();
- di.reset(identifierBytes, identifierBytes.length);
- final ContainerTokenIdentifier tokenId = new ContainerTokenIdentifier();
- tokenId.readFields(di);
-
- Token token = new Token(
- identifierBytes, containerToken.getPassword().array(), new Text(
- containerToken.getKind()), new Text(containerToken.getService()));
-
- unauthorizedUser.addToken(token);
- ContainerManager client =
- unauthorizedUser.doAs(new PrivilegedAction() {
- @Override
- public ContainerManager run() {
- ContainerManager client = (ContainerManager) yarnRPC.getProxy(
- ContainerManager.class, NetUtils
- .createSocketAddr(allocatedContainer.getNodeId().toString()),
- conf);
-
- LOG.info("Going to contact NM: unauthorized request");
-
- callWithIllegalContainerID(client, tokenId, allocatedContainer);
- callWithIllegalResource(client, tokenId, allocatedContainer);
- // UserName is no longer sent using containerLaunchContext.
-// callWithIllegalUserName(client, tokenId, allocatedContainer);
-
- return client;
- }
- });
-
- // ///////// End of testing for illegal containerIDs, illegal Resources and
- // illegal users
+ final ContainerTokenIdentifier tokenId =
+ BuilderUtils.newContainerTokenIdentifier(containerToken);
/////////// Test calls with expired tokens
- RPC.stopProxy(client);
- unauthorizedUser = UserGroupInformation
+ UserGroupInformation unauthorizedUser = UserGroupInformation
.createRemoteUser(containerID.toString());
RMContainerTokenSecretManager containerTokenSecreteManager =
@@ -349,9 +347,10 @@ public ContainerManager run() {
containerTokenSecreteManager.createPassword(
newTokenId);
// Create a valid token by using the key from the RM.
- token = new Token(
- newTokenId.getBytes(), passowrd, new Text(
- containerToken.getKind()), new Text(containerToken.getService()));
+ Token token =
+ new Token(newTokenId.getBytes(), passowrd,
+ new Text(containerToken.getKind()), new Text(
+ containerToken.getService()));
unauthorizedUser.addToken(token);
unauthorizedUser.doAs(new PrivilegedAction() {
@@ -369,7 +368,7 @@ public Void run() {
request.setContainerLaunchContext(context);
allocatedContainer.setContainerToken(BuilderUtils.newContainerToken(
allocatedContainer.getNodeId(), passowrd, newTokenId));
- request.setContainer(allocatedContainer);
+ request.setContainerToken(allocatedContainer.getContainerToken());
//Calling startContainer with an expired token.
try {
@@ -524,93 +523,6 @@ private Container requestAndGetContainer(AMRMProtocol scheduler,
return allocatedContainers.get(0);
}
- void callWithIllegalContainerID(ContainerManager client,
- ContainerTokenIdentifier tokenId, Container container) {
- StartContainerRequest request = recordFactory
- .newRecordInstance(StartContainerRequest.class);
- ContainerLaunchContext context =
- createContainerLaunchContextForTest(tokenId);
- ContainerId newContainerId = BuilderUtils.newContainerId(BuilderUtils
- .newApplicationAttemptId(tokenId.getContainerID()
- .getApplicationAttemptId().getApplicationId(), 1), 42);
- ContainerId oldContainerId = container.getId();
- try {
- container.setId(newContainerId);
- request.setContainer(container);
- request.setContainerLaunchContext(context);
- client.startContainer(request);
- fail("Connection initiation with unauthorized "
- + "access is expected to fail.");
- } catch (YarnRemoteException e) {
- LOG.info("Got exception : ", e);
- Assert.assertTrue(e.getMessage().contains(
- "Unauthorized request to start container. "
- + "\nExpected containerId: " + tokenId.getContainerID()
- + " Found: " + newContainerId.toString()));
- } catch (IOException e) {
- LOG.info("Got IOException: ",e);
- fail("IOException is not expected.");
- } finally {
- container.setId(oldContainerId);
- }
- }
-
- void callWithIllegalResource(ContainerManager client,
- ContainerTokenIdentifier tokenId, Container container) {
- StartContainerRequest request = recordFactory
- .newRecordInstance(StartContainerRequest.class);
- // Authenticated but unauthorized, due to wrong resource
- ContainerLaunchContext context =
- createContainerLaunchContextForTest(tokenId);
- Resource rsrc = container.getResource();
- container.setResource(BuilderUtils.newResource(2048, 1));
- request.setContainerLaunchContext(context);
- request.setContainer(container);
- try {
- client.startContainer(request);
- fail("Connection initiation with unauthorized "
- + "access is expected to fail.");
- } catch (YarnRemoteException e) {
- LOG.info("Got exception : ", e);
- Assert.assertTrue(e.getMessage().contains(
- "Unauthorized request to start container. "));
- Assert.assertTrue(e.getMessage().contains(
- "\nExpected resource " + tokenId.getResource().toString()
- + " but found " + container.getResource().toString()));
- } catch (IOException e) {
- LOG.info("Got IOException: ",e);
- fail("IOException is not expected.");
- }
- container.setResource(rsrc);
- }
-
- void callWithIllegalUserName(ContainerManager client,
- ContainerTokenIdentifier tokenId, Container container) {
- StartContainerRequest request = recordFactory
- .newRecordInstance(StartContainerRequest.class);
- // Authenticated but unauthorized, due to wrong resource
- ContainerLaunchContext context =
- createContainerLaunchContextForTest(tokenId);
- String user = "invalidUser";
- request.setContainerLaunchContext(context);
- request.setContainer(container);
- try {
- client.startContainer(request);
- fail("Connection initiation with unauthorized "
- + "access is expected to fail.");
- } catch (YarnRemoteException e) {
- LOG.info("Got exception : ", e);
- Assert.assertTrue(e.getMessage().contains(
- "Unauthorized request to start container. "));
- Assert.assertTrue(e.getMessage().contains(
- "Expected user-name " + tokenId.getApplicationSubmitter()
- + " but found " + user));
- } catch (IOException e) {
- LOG.info("Got IOException: ",e);
- fail("IOException is not expected.");
- }
- }
-
private ContainerLaunchContext createContainerLaunchContextForTest(
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =