diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
index 946d9c6..cbb8e72 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.Clock;
@@ -61,4 +62,6 @@
Set getBlacklistedNodes();
ClientToAMTokenSecretManager getClientToAMTokenSecretManager();
+
+ Map getNMTokens();
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 8abd58d..71d5b52 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -886,6 +886,8 @@ protected void serviceStop() throws Exception {
private final Configuration conf;
private final ClusterInfo clusterInfo = new ClusterInfo();
private final ClientToAMTokenSecretManager clientToAMTokenSecretManager;
+ private final ConcurrentHashMap nmTokens =
+ new ConcurrentHashMap();
public RunningAppContext(Configuration config) {
this.conf = config;
@@ -952,6 +954,11 @@ public ClusterInfo getClusterInfo() {
public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
return clientToAMTokenSecretManager;
}
+
+ @Override
+ public Map getNMTokens() {
+ return this.nmTokens;
+ }
}
@SuppressWarnings("unchecked")
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
index 283fad0..12d4be9 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.mapreduce.v2.app.launcher;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.security.PrivilegedAction;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
@@ -35,7 +33,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -55,6 +52,8 @@
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -73,22 +72,22 @@
private ConcurrentHashMap containers =
new ConcurrentHashMap();
- private AppContext context;
+ private final AppContext context;
protected ThreadPoolExecutor launcherPool;
protected static final int INITIAL_POOL_SIZE = 10;
private int limitOnPoolSize;
private Thread eventHandlingThread;
protected BlockingQueue eventQueue =
new LinkedBlockingQueue();
- YarnRPC rpc;
private final AtomicBoolean stopped;
+ private ContainerManagementProtocolProxy cmProxy;
private Container getContainer(ContainerLauncherEvent event) {
ContainerId id = event.getContainerID();
Container c = containers.get(id);
if(c == null) {
c = new Container(event.getTaskAttemptID(), event.getContainerID(),
- event.getContainerMgrAddress(), event.getContainerToken());
+ event.getContainerMgrAddress());
Container old = containers.putIfAbsent(id, c);
if(old != null) {
c = old;
@@ -114,16 +113,13 @@ private void removeContainerIfDone(ContainerId id) {
private TaskAttemptId taskAttemptID;
private ContainerId containerID;
final private String containerMgrAddress;
- private org.apache.hadoop.yarn.api.records.Token containerToken;
public Container(TaskAttemptId taId, ContainerId containerID,
- String containerMgrAddress,
- org.apache.hadoop.yarn.api.records.Token containerToken) {
+ String containerMgrAddress) {
this.state = ContainerState.PREP;
this.taskAttemptID = taId;
this.containerMgrAddress = containerMgrAddress;
this.containerID = containerID;
- this.containerToken = containerToken;
}
public synchronized boolean isCompletelyDone() {
@@ -140,11 +136,10 @@ public synchronized void launch(ContainerRemoteLaunchEvent event) {
return;
}
- ContainerManagementProtocol proxy = null;
+ ContainerManagementProtocolProxyData proxy = null;
try {
- proxy = getCMProxy(containerID, containerMgrAddress,
- containerToken);
+ proxy = getCMProxy(containerMgrAddress, containerID);
// Construct the actual Container
ContainerLaunchContext containerLaunchContext =
@@ -155,7 +150,8 @@ public synchronized void launch(ContainerRemoteLaunchEvent event) {
.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainerToken(event.getContainerToken());
- StartContainerResponse response = proxy.startContainer(startRequest);
+ StartContainerResponse response =
+ proxy.getContainerManagementProtocol().startContainer(startRequest);
ByteBuffer portInfo =
response.getAllServicesMetaData().get(
@@ -185,7 +181,7 @@ public synchronized void launch(ContainerRemoteLaunchEvent event) {
sendContainerLaunchFailedMsg(taskAttemptID, message);
} finally {
if (proxy != null) {
- ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
+ cmProxy.mayBeCloseProxy(proxy);
}
}
}
@@ -198,29 +194,30 @@ public synchronized void kill() {
} else if (!isCompletelyDone()) {
LOG.info("KILLING " + taskAttemptID);
- ContainerManagementProtocol proxy = null;
+ ContainerManagementProtocolProxyData proxy = null;
try {
- proxy = getCMProxy(this.containerID, this.containerMgrAddress,
- this.containerToken);
+ proxy = getCMProxy(this.containerMgrAddress, this.containerID);
- // kill the remote container if already launched
- StopContainerRequest stopRequest = Records
+ // kill the remote container if already launched
+ StopContainerRequest stopRequest = Records
.newRecord(StopContainerRequest.class);
- stopRequest.setContainerId(this.containerID);
- proxy.stopContainer(stopRequest);
+ stopRequest.setContainerId(this.containerID);
+ proxy.getContainerManagementProtocol().stopContainer(stopRequest);
} catch (Throwable t) {
// ignore the cleanup failure
String message = "cleanup failed for container "
- + this.containerID + " : "
- + StringUtils.stringifyException(t);
- context.getEventHandler().handle(
- new TaskAttemptDiagnosticsUpdateEvent(this.taskAttemptID, message));
+ + this.containerID + " : "
+ + StringUtils.stringifyException(t);
+ context.getEventHandler()
+ .handle(
+ new TaskAttemptDiagnosticsUpdateEvent(this.taskAttemptID,
+ message));
LOG.warn(message);
} finally {
if (proxy != null) {
- ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
+ cmProxy.mayBeCloseProxy(proxy);
}
}
this.state = ContainerState.DONE;
@@ -239,21 +236,14 @@ public ContainerLauncherImpl(AppContext context) {
}
@Override
- protected void serviceInit(Configuration config) throws Exception {
- Configuration conf = new Configuration(config);
- conf.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
- 0);
+ protected void serviceInit(Configuration conf) throws Exception {
this.limitOnPoolSize = conf.getInt(
MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
- this.rpc = createYarnRPC(conf);
super.serviceInit(conf);
- }
-
- protected YarnRPC createYarnRPC(Configuration conf) {
- return YarnRPC.create(conf);
+ cmProxy =
+ new ContainerManagementProtocolProxy(conf, context.getNMTokens());
}
protected void serviceStart() throws Exception {
@@ -348,34 +338,6 @@ protected EventProcessor createEventProcessor(ContainerLauncherEvent event) {
return new EventProcessor(event);
}
- protected ContainerManagementProtocol getCMProxy(ContainerId containerID,
- final String containerManagerBindAddr,
- org.apache.hadoop.yarn.api.records.Token containerToken)
- throws IOException {
-
- final InetSocketAddress cmAddr =
- NetUtils.createSocketAddr(containerManagerBindAddr);
-
- // the user in createRemoteUser in this context has to be ContainerID
- UserGroupInformation user =
- UserGroupInformation.createRemoteUser(containerID.toString());
-
- Token token =
- ConverterUtils.convertFromYarn(containerToken, cmAddr);
- user.addToken(token);
-
- ContainerManagementProtocol proxy = user
- .doAs(new PrivilegedAction() {
- @Override
- public ContainerManagementProtocol run() {
- return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class,
- cmAddr, getConfig());
- }
- });
- return proxy;
- }
-
-
/**
* Setup and start the container on remote nodemanager.
*/
@@ -410,7 +372,7 @@ public void run() {
removeContainerIfDone(containerID);
}
}
-
+
@SuppressWarnings("unchecked")
void sendContainerLaunchFailedMsg(TaskAttemptId taskAttemptID,
String message) {
@@ -430,4 +392,9 @@ public void handle(ContainerLauncherEvent event) {
throw new YarnRuntimeException(e);
}
}
+
+ public ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
+ String containerMgrBindAddr, ContainerId containerId) throws IOException {
+ return cmProxy.getProxy(containerMgrBindAddr, containerId);
+ }
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index ec8480a..1263582 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -63,6 +63,7 @@
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -584,6 +585,14 @@ public void rampDownReduces(int rampDown) {
}
int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
List newContainers = response.getAllocatedContainers();
+ // Setting NMTokens
+ if (response.getNMTokens() != null) {
+ for (NMToken nmToken : response.getNMTokens()) {
+ getContext().getNMTokens().put(nmToken.getNodeId().toString(),
+ nmToken.getToken());
+ }
+ }
+
List finishedContainers = response.getCompletedContainersStatuses();
if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
//something changed
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
index f4fc42f..521e282 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
@@ -130,4 +131,10 @@ public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
// Not implemented
return null;
}
+
+ @Override
+ public Map getNMTokens() {
+ // Not Implemented
+ return null;
+ }
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
index 0bf6305..48cb641 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.junit.Test;
/**
@@ -225,8 +226,8 @@ public void handle(ContainerLauncherEvent event) {
}
@Override
- protected ContainerManagementProtocol getCMProxy(ContainerId contianerID,
- String containerManagerBindAddr, Token containerToken)
+ public ContainerManagementProtocolProxyData getCMProxy(
+ String containerMgrBindAddr, ContainerId containerId)
throws IOException {
try {
synchronized (this) {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index 1c450fb..1742d90 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -68,6 +68,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -857,7 +858,13 @@ public ClusterInfo getClusterInfo() {
return null;
}
+ @Override
public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
+ return null;
+ }
+
+ @Override
+ public Map getNMTokens() {
// Not Implemented
return null;
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index e33ffd0..5b87a08 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -62,12 +62,15 @@
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.junit.Test;
public class TestContainerLauncher {
@@ -342,16 +345,26 @@ public MRAppWithSlowNM() {
}
@Override
- protected ContainerLauncher createContainerLauncher(AppContext context) {
+ protected ContainerLauncher
+ createContainerLauncher(final AppContext context) {
return new ContainerLauncherImpl(context) {
+
@Override
- protected ContainerManagementProtocol getCMProxy(ContainerId containerID,
- String containerManagerBindAddr, Token containerToken)
+ public ContainerManagementProtocolProxyData getCMProxy(
+ String containerMgrBindAddr, ContainerId containerId)
throws IOException {
- // make proxy connect to our local containerManager server
- ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(
- ContainerManagementProtocol.class,
- NetUtils.getConnectAddress(server), conf);
+ Token dummyToken =
+ Token.newInstance("NMTokenIdentifier".getBytes(),
+ NMTokenIdentifier.KIND.toString(), "password".getBytes(),
+ "NMToken");
+ ContainerManagementProtocolProxy cmProxy =
+ new ContainerManagementProtocolProxy(conf, context.getNMTokens());
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ ContainerManagementProtocolProxyData proxy =
+ cmProxy.new ContainerManagementProtocolProxyData(
+ YarnRPC.create(conf),
+ addr.getHostName() + ":" + addr.getPort(), containerId,
+ dummyToken);
return proxy;
}
};
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index dcd18dc..cc12ce3 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -95,11 +95,6 @@ public ContainerLauncherImplUnderTest(AppContext context, YarnRPC rpc) {
this.rpc = rpc;
}
- @Override
- protected YarnRPC createYarnRPC(Configuration conf) {
- return rpc;
- }
-
public void waitForPoolToIdle() throws InterruptedException {
//I wish that we did not need the sleep, but it is here so that we are sure
// That the other thread had time to insert the event into the queue and
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
index 2c1f3a2..8605bb4 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -315,4 +316,10 @@ public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
// Not implemented.
return null;
}
+
+ @Override
+ public Map getNMTokens() {
+ // Not Implemented.
+ return null;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 729a174..4af7745 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -728,6 +728,21 @@
YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size";
public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500;
+ /**
+ * Maximum number of proxy connections for node manager. It should always be
+ * more than 1. NMClient and MRAppMaster will use this to cache connection
+ * with node manager. There will be at max one connection per node manager.
+ * Ex. configuring it to a value of 5 will make sure that client will at
+ * max have 5 connections cached with 5 different node managers. These
+ * connections will be timed out if idle for more than system wide idle
+ * timeout period. The token if used for authentication then it will be used
+ * only at connection creation time. If new token is received then earlier
+ * connection should be closed in order to use newer token.
+ */
+ public static final String NM_CLIENT_MAX_NM_PROXIES =
+ YARN_PREFIX + "client.max-nodemanagers-proxies";
+ public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 10;
+
public YarnConfiguration() {
super();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 6b7768b..638ce13 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -70,6 +70,7 @@
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -447,7 +448,8 @@ public boolean run() throws YarnException, IOException {
resourceManager.start();
containerListener = new NMCallbackHandler();
- nmClientAsync = NMClientAsync.createNMClientAsync(containerListener);
+ nmClientAsync =
+ new NMClientAsyncImpl(containerListener, resourceManager.getNMTokens());
nmClientAsync.init(conf);
nmClientAsync.start();
@@ -683,8 +685,7 @@ public void onContainerStarted(ContainerId containerId,
}
Container container = containers.get(containerId);
if (container != null) {
- nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId(),
- container.getContainerToken());
+ nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 992691e..df333d2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -26,6 +26,7 @@
import java.net.URL;
import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +34,8 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.junit.AfterClass;
@@ -50,7 +53,7 @@
protected static String APPMASTER_JAR = JarFinder.getJar(ApplicationMaster.class);
@BeforeClass
- public static void setup() throws InterruptedException, IOException {
+ public static void setup() throws InterruptedException, Exception {
LOG.info("Starting up YARN cluster");
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
conf.setClass(YarnConfiguration.RM_SCHEDULER,
@@ -60,6 +63,9 @@ public static void setup() throws InterruptedException, IOException {
TestDistributedShell.class.getSimpleName(), 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
+ NodeManager nm = yarnCluster.getNodeManager(0);
+ waitForNMToRegister(nm);
+
URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
if (url == null) {
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
@@ -195,5 +201,14 @@ public void testDSShellWithInvalidArgs() throws Exception {
}
}
+ protected static void waitForNMToRegister(NodeManager nm)
+ throws Exception {
+ int attempt = 60;
+ ContainerManagerImpl cm =
+ ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
+ while (cm.getBlockNewContainerRequestsStatus() && attempt-- > 0) {
+ Thread.sleep(2000);
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ContainerManagementProtocolProxy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ContainerManagementProtocolProxy.java
new file mode 100644
index 0000000..7378522
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ContainerManagementProtocolProxy.java
@@ -0,0 +1,219 @@
+package org.apache.hadoop.yarn.client;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+
+/**
+ * Helper class to manage container manager proxies
+ */
+@LimitedPrivate({ "MapReduce", "YARN" })
+public class ContainerManagementProtocolProxy {
+ static final Log LOG = LogFactory.getLog(ContainerManagementProtocolProxy.class);
+
+ private final int maxConnectedNMs;
+ private final LinkedHashMap cmProxy;
+ private Map nmTokens;
+ private final Configuration conf;
+ private final YarnRPC rpc;
+
+ public ContainerManagementProtocolProxy(Configuration conf,
+ Map nmTokens) {
+ this.nmTokens = nmTokens;
+ this.conf = conf;
+
+ maxConnectedNMs =
+ conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
+ YarnConfiguration.DEFAULT_NM_CLIENT_MAX_NM_PROXIES);
+ if (maxConnectedNMs < 1) {
+ throw new YarnRuntimeException(
+ YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES
+ + " (" + maxConnectedNMs + ") can not be less than 1.");
+ }
+ LOG.info(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : "
+ + maxConnectedNMs);
+
+ cmProxy =
+ new LinkedHashMap();
+ rpc = YarnRPC.create(conf);
+ }
+
+ public synchronized ContainerManagementProtocolProxyData getProxy(
+ String containerManagerBindAddr, ContainerId containerId)
+ throws InvalidToken {
+
+ // This get call will update the map which is working as LRU cache.
+ ContainerManagementProtocolProxyData proxy =
+ cmProxy.get(containerManagerBindAddr);
+
+ while (proxy != null
+ && !proxy.token.getIdentifier().equals(
+ nmTokens.get(containerManagerBindAddr).getIdentifier())) {
+ LOG.info("Refreshing proxy as NMToken got updated for node : "
+ + containerManagerBindAddr);
+ // Token is updated. check if anyone has already tried closing it.
+ if (!proxy.scheduledForClose) {
+ // try closing the proxy. Here if someone is already using it
+ // then we might not close it. In which case we will wait.
+ removeProxy(proxy);
+ } else {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ if (proxy.activeCallers < 0) {
+ proxy = cmProxy.get(containerManagerBindAddr);
+ }
+ }
+
+ if (proxy == null) {
+ proxy =
+ new ContainerManagementProtocolProxyData(rpc, containerManagerBindAddr,
+ containerId, nmTokens.get(containerManagerBindAddr));
+ if (cmProxy.size() > maxConnectedNMs) {
+ // Number of existing proxy exceed the limit.
+ String cmAddr = cmProxy.keySet().iterator().next();
+ removeProxy(cmProxy.get(cmAddr));
+ }
+
+ cmProxy.put(containerManagerBindAddr, proxy);
+ }
+ // This is to track active users of this proxy.
+ proxy.activeCallers++;
+ updateLRUCache(containerManagerBindAddr);
+
+ return proxy;
+ }
+
+ private void updateLRUCache(String containerManagerBindAddr) {
+ ContainerManagementProtocolProxyData proxy =
+ cmProxy.remove(containerManagerBindAddr);
+ cmProxy.put(containerManagerBindAddr, proxy);
+ }
+
+ public synchronized void mayBeCloseProxy(
+ ContainerManagementProtocolProxyData proxy) {
+ proxy.activeCallers--;
+ if (proxy.scheduledForClose && proxy.activeCallers < 0) {
+ LOG.info("Closing proxy : " + proxy.containerManagerBindAddr);
+ cmProxy.remove(proxy.containerManagerBindAddr);
+ try {
+ rpc.stopProxy(proxy.getContainerManagementProtocol(), conf);
+ } finally {
+ this.notifyAll();
+ }
+ }
+ }
+
+ private synchronized void removeProxy(
+ ContainerManagementProtocolProxyData proxy) {
+ if (!proxy.scheduledForClose) {
+ proxy.scheduledForClose = true;
+ mayBeCloseProxy(proxy);
+ }
+ }
+
+ public synchronized void stopAllProxies() {
+ List nodeIds = new ArrayList();
+ nodeIds.addAll(this.cmProxy.keySet());
+ for (String nodeId : nodeIds) {
+ ContainerManagementProtocolProxyData proxy = cmProxy.get(nodeId);
+ // Explicitly reducing the proxy count to allow stopping proxy.
+ proxy.activeCallers = 0;
+ try {
+ removeProxy(proxy);
+ } catch (Throwable t) {
+ LOG.error("Error closing connection", t);
+ }
+ }
+ cmProxy.clear();
+ }
+
+ public void setNMTokens(Map nmTokens) {
+ this.nmTokens = nmTokens;
+ }
+
+ public class ContainerManagementProtocolProxyData {
+ private final String containerManagerBindAddr;
+ private final ContainerManagementProtocol proxy;
+ private int activeCallers;
+ private boolean scheduledForClose;
+ private final Token token;
+
+ @Private
+ @VisibleForTesting
+ public ContainerManagementProtocolProxyData(YarnRPC rpc,
+ String containerManagerBindAddr,
+ ContainerId containerId, Token token) throws InvalidToken {
+ this.containerManagerBindAddr = containerManagerBindAddr;
+ ;
+ this.activeCallers = 0;
+ this.scheduledForClose = false;
+ this.token = token;
+ this.proxy = newProxy(rpc, containerManagerBindAddr, containerId, token);
+ }
+
+ @Private
+ @VisibleForTesting
+ protected ContainerManagementProtocol newProxy(final YarnRPC rpc,
+ String containerManagerBindAddr, ContainerId containerId, Token token)
+ throws InvalidToken {
+ if (token == null) {
+ throw new InvalidToken("No NMToken sent for "
+ + containerManagerBindAddr);
+ }
+ final InetSocketAddress cmAddr =
+ NetUtils.createSocketAddr(containerManagerBindAddr);
+ LOG.info("Opening proxy : " + containerManagerBindAddr);
+ // the user in createRemoteUser in this context has to be ContainerID
+ UserGroupInformation user =
+ UserGroupInformation.createRemoteUser(containerId
+ .getApplicationAttemptId().toString());
+
+ org.apache.hadoop.security.token.Token nmToken =
+ ConverterUtils.convertFromYarn(token, cmAddr);
+ user.addToken(nmToken);
+
+ ContainerManagementProtocol proxy = user
+ .doAs(new PrivilegedAction() {
+
+ @Override
+ public ContainerManagementProtocol run() {
+ return (ContainerManagementProtocol) rpc.getProxy(
+ ContainerManagementProtocol.class, cmAddr, conf);
+ }
+ });
+ return proxy;
+ }
+
+ public ContainerManagementProtocol getContainerManagementProtocol() {
+ return proxy;
+ }
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
index e628745..08ea840 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
@@ -1,4 +1,5 @@
/**
+
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,6 +22,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -44,8 +46,8 @@
* Create a new instance of NMClient.
*/
@Public
- public static NMClient createNMClient() {
- NMClient client = new NMClientImpl();
+ public static NMClient createNMClient(ConcurrentMap nmTokens) {
+ NMClient client = new NMClientImpl(nmTokens);
return client;
}
@@ -53,8 +55,9 @@ public static NMClient createNMClient() {
* Create a new instance of NMClient.
*/
@Public
- public static NMClient createNMClient(String name) {
- NMClient client = new NMClientImpl(name);
+ public static NMClient createNMClient(String name,
+ ConcurrentMap nmTokens) {
+ NMClient client = new NMClientImpl(name, nmTokens);
return client;
}
@@ -89,35 +92,33 @@ protected NMClient(String name) {
*
* @param containerId the Id of the started container
* @param nodeId the Id of the NodeManager
- * @param containerToken the security token to verify authenticity of the
- * started container
+ *
* @throws YarnException
* @throws IOException
*/
- public abstract void stopContainer(ContainerId containerId, NodeId nodeId,
- Token containerToken) throws YarnException, IOException;
+ public abstract void stopContainer(ContainerId containerId, NodeId nodeId)
+ throws YarnException, IOException;
/**
* Query the status of a container.
*
* @param containerId the Id of the started container
* @param nodeId the Id of the NodeManager
- * @param containerToken the security token to verify authenticity of the
- * started container
+ *
* @return the status of a container
* @throws YarnException
* @throws IOException
*/
- public abstract ContainerStatus getContainerStatus(ContainerId containerId, NodeId nodeId,
- Token containerToken) throws YarnException, IOException;
+ public abstract ContainerStatus getContainerStatus(ContainerId containerId,
+ NodeId nodeId) throws YarnException, IOException;
/**
* Set whether the containers that are started by this client, and are
* still running should be stopped when the client stops. By default, the
- * feature should be enabled.
+ * feature should be enabled.
However, containers will be stopped only
+ * when service is stopped. i.e. after {@link NMClient#stop()}.
*
* @param enabled whether the feature is enabled or not
*/
public abstract void cleanupRunningContainersOnStop(boolean enabled);
-
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
index 6b9b7ee..507f8d9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -110,16 +111,19 @@
protected NMClient client;
protected CallbackHandler callbackHandler;
- public static NMClientAsync createNMClientAsync(CallbackHandler callbackHandler) {
- return new NMClientAsyncImpl(callbackHandler);
+ public static NMClientAsync createNMClientAsync(
+ CallbackHandler callbackHandler, ConcurrentMap nmTokens) {
+ return new NMClientAsyncImpl(callbackHandler, nmTokens);
}
- protected NMClientAsync(CallbackHandler callbackHandler) {
- this (NMClientAsync.class.getName(), callbackHandler);
+ protected NMClientAsync(CallbackHandler callbackHandler,
+ ConcurrentMap nmTokens) {
+ this (NMClientAsync.class.getName(), callbackHandler, nmTokens);
}
- protected NMClientAsync(String name, CallbackHandler callbackHandler) {
- this (name, new NMClientImpl(), callbackHandler);
+ protected NMClientAsync(String name, CallbackHandler callbackHandler,
+ ConcurrentMap nmTokens) {
+ this (name, new NMClientImpl(nmTokens), callbackHandler);
}
@Private
@@ -135,10 +139,10 @@ public abstract void startContainerAsync(
Container container, ContainerLaunchContext containerLaunchContext);
public abstract void stopContainerAsync(
- ContainerId containerId, NodeId nodeId, Token containerToken);
+ ContainerId containerId, NodeId nodeId);
public abstract void getContainerStatusAsync(
- ContainerId containerId, NodeId nodeId, Token containerToken);
+ ContainerId containerId, NodeId nodeId);
public NMClient getClient() {
return client;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
index 0622b2d..7f7df1a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
@@ -82,12 +82,14 @@
protected ConcurrentMap containers =
new ConcurrentHashMap();
- public NMClientAsyncImpl(CallbackHandler callbackHandler) {
- this (NMClientAsyncImpl.class.getName(), callbackHandler);
+ public NMClientAsyncImpl(CallbackHandler callbackHandler,
+ ConcurrentMap nmTokens) {
+ this(NMClientAsync.class.getName(), callbackHandler, nmTokens);
}
- public NMClientAsyncImpl(String name, CallbackHandler callbackHandler) {
- this (name, new NMClientImpl(), callbackHandler);
+ public NMClientAsyncImpl(String name, CallbackHandler callbackHandler,
+ ConcurrentMap nmTokens) {
+ this(name, new NMClientImpl(nmTokens), callbackHandler);
}
@Private
@@ -229,15 +231,14 @@ public void startContainerAsync(
}
}
- public void stopContainerAsync(ContainerId containerId, NodeId nodeId,
- Token containerToken) {
+ public void stopContainerAsync(ContainerId containerId, NodeId nodeId) {
if (containers.get(containerId) == null) {
callbackHandler.onStopContainerError(containerId,
RPCUtil.getRemoteException("Container " + containerId +
" is neither started nor scheduled to start"));
}
try {
- events.put(new ContainerEvent(containerId, nodeId, containerToken,
+ events.put(new ContainerEvent(containerId, nodeId, null,
ContainerEventType.STOP_CONTAINER));
} catch (InterruptedException e) {
LOG.warn("Exception when scheduling the event of stopping Container " +
@@ -246,10 +247,9 @@ public void stopContainerAsync(ContainerId containerId, NodeId nodeId,
}
}
- public void getContainerStatusAsync(ContainerId containerId, NodeId nodeId,
- Token containerToken) {
+ public void getContainerStatusAsync(ContainerId containerId, NodeId nodeId) {
try {
- events.put(new ContainerEvent(containerId, nodeId, containerToken,
+ events.put(new ContainerEvent(containerId, nodeId, null,
ContainerEventType.QUERY_CONTAINER));
} catch (InterruptedException e) {
LOG.warn("Exception when scheduling the event of querying the status" +
@@ -421,9 +421,9 @@ public ContainerState transition(
StatefulContainer container, ContainerEvent event) {
ContainerId containerId = event.getContainerId();
try {
- container.nmClientAsync.getClient().stopContainer(
- containerId, event.getNodeId(), event.getContainerToken());
- try {
+ container.nmClientAsync.getClient().stopContainer(
+ containerId, event.getNodeId());
+ try {
container.nmClientAsync.getCallbackHandler().onContainerStopped(
event.getContainerId());
} catch (Throwable thr) {
@@ -534,7 +534,7 @@ public void run() {
if (event.getType() == ContainerEventType.QUERY_CONTAINER) {
try {
ContainerStatus containerStatus = client.getContainerStatus(
- containerId, event.getNodeId(), event.getContainerToken());
+ containerId, event.getNodeId());
try {
callbackHandler.onContainerStatusReceived(
containerId, containerStatus);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
index bef0388..75a7773 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.security.PrivilegedAction;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -29,30 +27,24 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy;
+import org.apache.hadoop.yarn.client.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -91,14 +83,18 @@
new ConcurrentHashMap();
//enabled by default
- private final AtomicBoolean cleanupRunningContainers = new AtomicBoolean(true);
+ private final AtomicBoolean cleanupRunningContainers = new AtomicBoolean(true);
+ private ContainerManagementProtocolProxy cmProxy;
+ private ConcurrentMap nmTokens;
- public NMClientImpl() {
+ public NMClientImpl(ConcurrentMap nmTokens) {
super(NMClientImpl.class.getName());
+ this.nmTokens = nmTokens;
}
- public NMClientImpl(String name) {
+ public NMClientImpl(String name, ConcurrentMap nmTokens) {
super(name);
+ this.nmTokens = nmTokens;
}
@Override
@@ -108,6 +104,7 @@ protected void serviceStop() throws Exception {
if (getCleanupRunningContainers().get()) {
cleanupRunningContainers();
}
+ cmProxy.stopAllProxies();
super.serviceStop();
}
@@ -115,8 +112,7 @@ protected synchronized void cleanupRunningContainers() {
for (StartedContainer startedContainer : startedContainers.values()) {
try {
stopContainer(startedContainer.getContainerId(),
- startedContainer.getNodeId(),
- startedContainer.getContainerToken());
+ startedContainer.getNodeId());
} catch (YarnException e) {
LOG.error("Failed to stop Container " +
startedContainer.getContainerId() +
@@ -130,22 +126,28 @@ protected synchronized void cleanupRunningContainers() {
}
@Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ cmProxy =
+ new ContainerManagementProtocolProxy(conf, nmTokens);
+ }
+
+ @Override
public void cleanupRunningContainersOnStop(boolean enabled) {
getCleanupRunningContainers().set(enabled);
}
-
+
protected static class StartedContainer {
private ContainerId containerId;
private NodeId nodeId;
- private Token containerToken;
- private boolean stopped;
-
+ private ContainerState state;
+
+
public StartedContainer(ContainerId containerId, NodeId nodeId,
Token containerToken) {
this.containerId = containerId;
this.nodeId = nodeId;
- this.containerToken = containerToken;
- stopped = false;
+ state = ContainerState.NEW;
}
public ContainerId getContainerId() {
@@ -155,137 +157,17 @@ public ContainerId getContainerId() {
public NodeId getNodeId() {
return nodeId;
}
-
- public Token getContainerToken() {
- return containerToken;
- }
}
- protected static final class NMCommunicator extends AbstractService {
- private ContainerId containerId;
- private NodeId nodeId;
- private Token containerToken;
- private ContainerManagementProtocol containerManager;
-
- public NMCommunicator(ContainerId containerId, NodeId nodeId,
- Token containerToken) {
- super(NMCommunicator.class.getName());
- this.containerId = containerId;
- this.nodeId = nodeId;
- this.containerToken = containerToken;
- }
-
- @Override
- protected synchronized void serviceStart() throws Exception {
- final YarnRPC rpc = YarnRPC.create(getConfig());
-
- final InetSocketAddress containerAddress =
- NetUtils.createSocketAddr(nodeId.toString());
-
- // the user in createRemoteUser in this context has to be ContainerId
- UserGroupInformation currentUser =
- UserGroupInformation.createRemoteUser(containerId.toString());
-
- org.apache.hadoop.security.token.Token token =
- ConverterUtils.convertFromYarn(containerToken, containerAddress);
- currentUser.addToken(token);
-
- containerManager = currentUser
- .doAs(new PrivilegedAction() {
- @Override
- public ContainerManagementProtocol run() {
- return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class,
- containerAddress, getConfig());
- }
- });
-
- LOG.debug("Connecting to ContainerManager at " + containerAddress);
- super.serviceStart();
- }
-
- @Override
- protected synchronized void serviceStop() throws Exception {
- if (this.containerManager != null) {
- RPC.stopProxy(this.containerManager);
-
- if (LOG.isDebugEnabled()) {
- InetSocketAddress containerAddress =
- NetUtils.createSocketAddr(nodeId.toString());
- LOG.debug("Disconnecting from ContainerManager at " +
- containerAddress);
- }
- }
- super.serviceStop();
- }
-
- public synchronized Map startContainer(
- Container container, ContainerLaunchContext containerLaunchContext)
- throws YarnException, IOException {
- if (!container.getId().equals(containerId)) {
- throw new IllegalArgumentException(
- "NMCommunicator's containerId mismatches the given Container's");
- }
- StartContainerResponse startResponse = null;
- try {
- StartContainerRequest startRequest =
- Records.newRecord(StartContainerRequest.class);
- startRequest.setContainerToken(container.getContainerToken());
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startResponse = containerManager.startContainer(startRequest);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Started Container " + containerId);
- }
- } catch (YarnException e) {
- LOG.warn("Container " + containerId + " failed to start", e);
- throw e;
- } catch (IOException e) {
- LOG.warn("Container " + containerId + " failed to start", e);
- throw e;
- }
- return startResponse.getAllServicesMetaData();
- }
-
- public synchronized void stopContainer() throws YarnException,
- IOException {
- try {
- StopContainerRequest stopRequest =
- Records.newRecord(StopContainerRequest.class);
- stopRequest.setContainerId(containerId);
- containerManager.stopContainer(stopRequest);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stopped Container " + containerId);
- }
- } catch (YarnException e) {
- LOG.warn("Container " + containerId + " failed to stop", e);
- throw e;
- } catch (IOException e) {
- LOG.warn("Container " + containerId + " failed to stop", e);
- throw e;
- }
- }
-
- public synchronized ContainerStatus getContainerStatus()
- throws YarnException, IOException {
- GetContainerStatusResponse statusResponse = null;
- try {
- GetContainerStatusRequest statusRequest =
- Records.newRecord(GetContainerStatusRequest.class);
- statusRequest.setContainerId(containerId);
- statusResponse = containerManager.getContainerStatus(statusRequest);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Got the status of Container " + containerId);
- }
- } catch (YarnException e) {
- LOG.warn(
- "Unable to get the status of Container " + containerId, e);
- throw e;
- } catch (IOException e) {
- LOG.warn(
- "Unable to get the status of Container " + containerId, e);
- throw e;
- }
- return statusResponse.getStatus();
+ private void addStartingContainer(StartedContainer startedContainer)
+ throws YarnException {
+ if (startedContainers.putIfAbsent(startedContainer.containerId,
+ startedContainer) != null) {
+ throw RPCUtil.getRemoteException("Container "
+ + startedContainer.containerId.toString() + " is already started");
}
+ startedContainers
+ .put(startedContainer.getContainerId(), startedContainer);
}
@Override
@@ -293,108 +175,112 @@ public synchronized ContainerStatus getContainerStatus()
Container container, ContainerLaunchContext containerLaunchContext)
throws YarnException, IOException {
// Do synchronization on StartedContainer to prevent race condition
- // between startContainer and stopContainer
- synchronized (addStartedContainer(container)) {
+ // between startContainer and stopContainer only when startContainer is
+ // in progress for a given container.
+ StartedContainer startingContainer = createStartedContainer(container);
+ synchronized (startingContainer) {
+ addStartingContainer(startingContainer);
+
Map allServiceResponse;
- NMCommunicator nmCommunicator = null;
+ ContainerManagementProtocolProxyData proxy = null;
try {
- nmCommunicator = new NMCommunicator(container.getId(),
- container.getNodeId(), container.getContainerToken());
- nmCommunicator.init(getConfig());
- nmCommunicator.start();
+ proxy =
+ cmProxy.getProxy(container.getNodeId().toString(),
+ container.getId());
allServiceResponse =
- nmCommunicator.startContainer(container, containerLaunchContext);
+ proxy
+ .getContainerManagementProtocol().startContainer(
+ StartContainerRequest.newInstance(containerLaunchContext,
+ container.getContainerToken())).getAllServicesMetaData();
+ startingContainer.state = ContainerState.RUNNING;
} catch (YarnException e) {
+ startingContainer.state = ContainerState.COMPLETE;
// Remove the started container if it failed to start
- removeStartedContainer(container.getId());
+ removeStartedContainer(startingContainer);
throw e;
} catch (IOException e) {
- removeStartedContainer(container.getId());
+ startingContainer.state = ContainerState.COMPLETE;
+ removeStartedContainer(startingContainer);
throw e;
} catch (Throwable t) {
- removeStartedContainer(container.getId());
+ startingContainer.state = ContainerState.COMPLETE;
+ removeStartedContainer(startingContainer);
throw RPCUtil.getRemoteException(t);
} finally {
- if (nmCommunicator != null) {
- nmCommunicator.stop();
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
}
}
return allServiceResponse;
}
-
- // Three choices:
- // 1. starting and releasing the proxy before and after each interaction
- // 2. starting the proxy when starting the container and releasing it when
- // stopping the container
- // 3. starting the proxy when starting the container and releasing it when
- // stopping the client
- // Adopt 1 currently
}
@Override
- public void stopContainer(ContainerId containerId, NodeId nodeId,
- Token containerToken) throws YarnException, IOException {
+ public void stopContainer(ContainerId containerId, NodeId nodeId)
+ throws YarnException, IOException {
StartedContainer startedContainer = getStartedContainer(containerId);
- if (startedContainer == null) {
- throw RPCUtil.getRemoteException("Container " + containerId +
- " is either not started yet or already stopped");
- }
+
// Only allow one request of stopping the container to move forward
// When entering the block, check whether the precursor has already stopped
// the container
- synchronized (startedContainer) {
- if (startedContainer.stopped) {
- return;
- }
- NMCommunicator nmCommunicator = null;
- try {
- nmCommunicator =
- new NMCommunicator(containerId, nodeId, containerToken);
- nmCommunicator.init(getConfig());
- nmCommunicator.start();
- nmCommunicator.stopContainer();
- } finally {
- if (nmCommunicator != null) {
- nmCommunicator.stop();
+ if (startedContainer != null) {
+ synchronized (startedContainer) {
+ if (startedContainer.state != ContainerState.RUNNING) {
+ return;
}
- startedContainer.stopped = true;
- removeStartedContainer(containerId);
+ stopContainerInternal(containerId, nodeId);
+ // Only after successful
+ startedContainer.state = ContainerState.COMPLETE;
+ removeStartedContainer(startedContainer);
}
+ } else {
+ stopContainerInternal(containerId, nodeId);
}
+
}
@Override
public ContainerStatus getContainerStatus(ContainerId containerId,
- NodeId nodeId, Token containerToken)
- throws YarnException, IOException {
- NMCommunicator nmCommunicator = null;
+ NodeId nodeId) throws YarnException, IOException {
+
+ ContainerManagementProtocolProxyData proxy = null;
try {
- nmCommunicator = new NMCommunicator(containerId, nodeId, containerToken);
- nmCommunicator.init(getConfig());
- nmCommunicator.start();
- ContainerStatus containerStatus = nmCommunicator.getContainerStatus();
+ proxy = cmProxy.getProxy(nodeId.toString(), containerId);
+ ContainerStatus containerStatus =
+ proxy.getContainerManagementProtocol().getContainerStatus(
+ GetContainerStatusRequest.newInstance(containerId)).getStatus();
return containerStatus;
} finally {
- if (nmCommunicator != null) {
- nmCommunicator.stop();
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
}
}
}
- protected synchronized StartedContainer addStartedContainer(
- Container container) throws YarnException, IOException {
- if (startedContainers.containsKey(container.getId())) {
- throw RPCUtil.getRemoteException("Container " + container.getId() +
- " is already started");
+ private void stopContainerInternal(ContainerId containerId, NodeId nodeId)
+ throws IOException, YarnException {
+ ContainerManagementProtocolProxyData proxy = null;
+ try {
+ proxy = cmProxy.getProxy(nodeId.toString(), containerId);
+ proxy.getContainerManagementProtocol().stopContainer(
+ StopContainerRequest.newInstance(containerId));
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
}
+ }
+
+ protected synchronized StartedContainer createStartedContainer(
+ Container container) throws YarnException, IOException {
StartedContainer startedContainer = new StartedContainer(container.getId(),
container.getNodeId(), container.getContainerToken());
- startedContainers.put(startedContainer.getContainerId(), startedContainer);
return startedContainer;
}
- protected synchronized void removeStartedContainer(ContainerId containerId) {
- startedContainers.remove(containerId);
+ protected synchronized void
+ removeStartedContainer(StartedContainer container) {
+ startedContainers.remove(container.containerId);
}
protected synchronized StartedContainer getStartedContainer(
@@ -405,5 +291,4 @@ protected synchronized StartedContainer getStartedContainer(
public AtomicBoolean getCleanupRunningContainers() {
return cleanupRunningContainers;
}
-
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
index 4f65948..0815e31 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
@@ -232,10 +232,10 @@ public void onContainerStarted(ContainerId containerId,
actualStartSuccessArray.set(containerId.getId(), 1);
// move on to the following success tests
- asyncClient.getContainerStatusAsync(containerId, nodeId, containerToken);
+ asyncClient.getContainerStatusAsync(containerId, nodeId);
} else {
// move on to the following failure tests
- asyncClient.stopContainerAsync(containerId, nodeId, containerToken);
+ asyncClient.stopContainerAsync(containerId, nodeId);
}
// Shouldn't crash the test thread
@@ -253,7 +253,7 @@ public void onContainerStatusReceived(ContainerId containerId,
actualQuerySuccess.addAndGet(1);
actualQuerySuccessArray.set(containerId.getId(), 1);
// move on to the following success tests
- asyncClient.stopContainerAsync(containerId, nodeId, containerToken);
+ asyncClient.stopContainerAsync(containerId, nodeId);
// Shouldn't crash the test thread
throw new RuntimeException("Ignorable Exception");
@@ -290,7 +290,7 @@ public void onStartContainerError(ContainerId containerId, Throwable t) {
actualStartFailure.addAndGet(1);
actualStartFailureArray.set(containerId.getId() - expectedSuccess, 1);
// move on to the following failure tests
- asyncClient.getContainerStatusAsync(containerId, nodeId, containerToken);
+ asyncClient.getContainerStatusAsync(containerId, nodeId);
// Shouldn't crash the test thread
throw new RuntimeException("Ignorable Exception");
@@ -383,33 +383,30 @@ private NMClient mockNMClient(int mode)
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.emptyMap());
- when(client.getContainerStatus(any(ContainerId.class), any(NodeId.class),
- any(Token.class))).thenReturn(
+ when(client.getContainerStatus(any(ContainerId.class),
+ any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doNothing().when(client).stopContainer(any(ContainerId.class),
- any(NodeId.class), any(Token.class));
+ any(NodeId.class));
break;
case 1:
doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
.startContainer(any(Container.class),
any(ContainerLaunchContext.class));
doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
- .getContainerStatus(any(ContainerId.class), any(NodeId.class),
- any(Token.class));
+ .getContainerStatus(any(ContainerId.class), any(NodeId.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
- .stopContainer(any(ContainerId.class), any(NodeId.class),
- any(Token.class));
+ .stopContainer(any(ContainerId.class), any(NodeId.class));
break;
case 2:
when(client.startContainer(any(Container.class),
any(ContainerLaunchContext.class))).thenReturn(
Collections.emptyMap());
- when(client.getContainerStatus(any(ContainerId.class), any(NodeId.class),
- any(Token.class))).thenReturn(
+ when(client.getContainerStatus(any(ContainerId.class),
+ any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
- .stopContainer(any(ContainerId.class), any(NodeId.class),
- any(Token.class));
+ .stopContainer(any(ContainerId.class), any(NodeId.class));
}
return client;
}
@@ -437,8 +434,7 @@ public void run() {
t.start();
barrierA.await();
- asyncClient.stopContainerAsync(container.getId(), container.getNodeId(),
- container.getContainerToken());
+ asyncClient.stopContainerAsync(container.getId(), container.getNodeId());
barrierC.await();
Assert.assertFalse("Starting and stopping should be out of order",
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index 232903b..309b5af 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -29,6 +29,7 @@
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -46,10 +47,12 @@
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.NMClient;
@@ -75,7 +78,8 @@
List nodeReports = null;
ApplicationAttemptId attemptId = null;
int nodeCount = 3;
-
+ ConcurrentHashMap nmTokens;
+
@Before
public void setup() throws YarnException, IOException {
// start minicluster
@@ -140,6 +144,7 @@ public void setup() throws YarnException, IOException {
if (iterationsLeft == 0) {
fail("Application hasn't bee started");
}
+ nmTokens = new ConcurrentHashMap();
// start am rm client
rmClient =
@@ -151,7 +156,7 @@ public void setup() throws YarnException, IOException {
assertEquals(STATE.STARTED, rmClient.getServiceState());
// start am nm client
- nmClient = (NMClientImpl) NMClient.createNMClient();
+ nmClient = (NMClientImpl) NMClient.createNMClient(nmTokens);
nmClient.init(conf);
nmClient.start();
assertNotNull(nmClient);
@@ -194,14 +199,13 @@ public void testNMClientNoCleanupOnStop()
assertEquals(0, nmClient.startedContainers.size());
}
- @Test (timeout = 60000)
+ @Test (timeout = 200000)
public void testNMClient()
throws YarnException, IOException {
-
rmClient.registerApplicationMaster("Host", 10000, "");
testContainerManagement(nmClient, allocateContainers(rmClient, 5));
-
+
rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
// stop the running containers on close
@@ -243,6 +247,11 @@ public void testNMClient()
for(Container container : allocResponse.getAllocatedContainers()) {
containers.add(container);
}
+ if (!allocResponse.getNMTokens().isEmpty()) {
+ for (NMToken token : allocResponse.getNMTokens()) {
+ nmTokens.put(token.getNodeId().toString(), token.getToken());
+ }
+ }
if(allocatedContainerCount < containersRequestedAny) {
// sleep to let NM's heartbeat to RM and trigger allocations
sleep(1000);
@@ -261,8 +270,7 @@ private void testContainerManagement(NMClientImpl nmClient,
// getContainerStatus shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.getContainerStatus(container.getId(), container.getNodeId(),
- container.getContainerToken());
+ nmClient.getContainerStatus(container.getId(), container.getNodeId());
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -272,12 +280,11 @@ private void testContainerManagement(NMClientImpl nmClient,
// stopContainer shouldn't be called before startContainer,
// otherwise, an exception will be thrown
try {
- nmClient.stopContainer(container.getId(), container.getNodeId(),
- container.getContainerToken());
+ nmClient.stopContainer(container.getId(), container.getNodeId());
fail("Exception is expected");
} catch (YarnException e) {
if (!e.getMessage()
- .contains("is either not started yet or already stopped")) {
+ .contains("is not handled by this NodeManager")) {
throw (AssertionError)
(new AssertionError("Exception is not expected: " + e).initCause(
e));
@@ -306,8 +313,7 @@ private void testContainerManagement(NMClientImpl nmClient,
-1000);
try {
- nmClient.stopContainer(container.getId(), container.getNodeId(),
- container.getContainerToken());
+ nmClient.stopContainer(container.getId(), container.getNodeId());
} catch (YarnException e) {
throw (AssertionError)
(new AssertionError("Exception is not expected: " + e)
@@ -335,8 +341,7 @@ private void testGetContainerStatus(Container container, int index,
while (true) {
try {
ContainerStatus status = nmClient.getContainerStatus(
- container.getId(), container.getNodeId(),
- container.getContainerToken());
+ container.getId(), container.getNodeId());
// NodeManager may still need some time to get the stable
// container status
if (status.getState() == state) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java
index 1f14b2a..67dfddb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java
@@ -55,7 +55,7 @@ public TokenInfo getTokenInfo(Class> protocol, Configuration conf) {
@Override
public Class extends TokenSelector extends TokenIdentifier>>
value() {
- return ContainerTokenSelector.class;
+ return NMTokenSelector.class;
}
};
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java
index 6071ba3..55ddd0a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java
@@ -21,21 +21,17 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Token;
@Public
@Evolving
@@ -48,14 +44,14 @@
private ApplicationAttemptId appAttemptId;
private NodeId nodeId;
private String appSubmitter;
- private int masterKeyId;
+ private int keyId;
public NMTokenIdentifier(ApplicationAttemptId appAttemptId, NodeId nodeId,
String applicationSubmitter, int masterKeyId) {
this.appAttemptId = appAttemptId;
this.nodeId = nodeId;
this.appSubmitter = applicationSubmitter;
- this.masterKeyId = masterKeyId;
+ this.keyId = masterKeyId;
}
/**
@@ -76,8 +72,8 @@ public String getApplicationSubmitter() {
return appSubmitter;
}
- public int getMastKeyId() {
- return masterKeyId;
+ public int getKeyId() {
+ return keyId;
}
@Override
@@ -89,7 +85,7 @@ public void write(DataOutput out) throws IOException {
out.writeInt(appAttemptId.getAttemptId());
out.writeUTF(this.nodeId.toString());
out.writeUTF(this.appSubmitter);
- out.writeInt(this.masterKeyId);
+ out.writeInt(this.keyId);
}
@Override
@@ -101,7 +97,7 @@ public void readFields(DataInput in) throws IOException {
String[] hostAddr = in.readUTF().split(":");
nodeId = NodeId.newInstance(hostAddr[0], Integer.parseInt(hostAddr[1]));
appSubmitter = in.readUTF();
- masterKeyId = in.readInt();
+ keyId = in.readInt();
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
new file mode 100644
index 0000000..c57e4a2
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
@@ -0,0 +1,56 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security;
+
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+
+public class NMTokenSelector implements
+ TokenSelector {
+
+ private static final Log LOG = LogFactory
+ .getLog(NMTokenSelector.class);
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Token selectToken(Text service,
+ Collection> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token extends TokenIdentifier> token : tokens) {
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Looking for service: " + service + ". Current token is "
+ + token);
+ }
+ if (NMTokenIdentifier.KIND.equals(token.getKind()) &&
+ service.equals(token.getService())) {
+ return (Token) token;
+ }
+ }
+ return null;
+ }
+
+}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 427f920..d860461 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -15,3 +15,4 @@ org.apache.hadoop.yarn.security.ContainerTokenIdentifier
org.apache.hadoop.yarn.security.AMRMTokenIdentifier
org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier
+org.apache.hadoop.yarn.security.NMTokenIdentifier
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
index 1094541..01da1af 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
@@ -114,4 +114,38 @@ public MasterKey getCurrentKey() {
public NMTokenIdentifier createIdentifier() {
return new NMTokenIdentifier();
}
+
+ /**
+ * Helper function for creating NMTokens.
+ */
+ public Token createNMToken(ApplicationAttemptId applicationAttemptId,
+ NodeId nodeId, String applicationSubmitter) {
+ byte[] password;
+ NMTokenIdentifier identifier;
+
+ this.readLock.lock();
+ try {
+ identifier =
+ new NMTokenIdentifier(applicationAttemptId, nodeId,
+ applicationSubmitter, this.currentMasterKey.getMasterKey()
+ .getKeyId());
+ password = this.createPassword(identifier);
+ } finally {
+ this.readLock.unlock();
+ }
+ return newInstance(password, identifier);
+ }
+
+ public static Token newInstance(byte[] password,
+ NMTokenIdentifier identifier) {
+ NodeId nodeId = identifier.getNodeId();
+ // RPC layer client expects ip:port as service for tokens
+ InetSocketAddress addr =
+ NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
+ Token nmToken =
+ Token.newInstance(identifier.getBytes(),
+ NMTokenIdentifier.KIND.toString(), password, SecurityUtil
+ .buildTokenService(addr).toString());
+ return nmToken;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index f21a60f..b6c59b6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -27,6 +27,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -63,6 +64,8 @@
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Builder utilities to construct various objects.
*
@@ -152,7 +155,8 @@ public static Token newContainerToken(ContainerId cId, String host,
int port, String user, Resource r, long expiryTime, int masterKeyId,
byte[] password, long rmIdentifier) throws IOException {
ContainerTokenIdentifier identifier =
- new ContainerTokenIdentifier(cId, host, user, r, expiryTime,
+ new ContainerTokenIdentifier(cId, host + ":" + port, user, r,
+ expiryTime,
masterKeyId, rmIdentifier);
return newContainerToken(BuilderUtils.newNodeId(host, port), password,
identifier);
@@ -228,6 +232,8 @@ public static Token newClientToAMToken(byte[] identifier, String kind,
return newToken(Token.class, identifier, kind, password, service);
}
+ @Private
+ @VisibleForTesting
public static Token newContainerToken(NodeId nodeId,
byte[] password, ContainerTokenIdentifier tokenIdentifier) {
// RPC layer client expects ip:port as service for tokens
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index e5a64cd..8ec3a5a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -435,7 +435,7 @@ Dispatcher getNMDispatcher(){
}
@VisibleForTesting
- Context getNMContext() {
+ public Context getNMContext() {
return this.context;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 69a5006..1d5ab13 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -38,6 +39,7 @@
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.CompositeService;
@@ -65,6 +67,7 @@
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -234,7 +237,7 @@ protected void serviceStart() throws Exception {
server =
rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, conf,
- this.context.getContainerTokenSecretManager(),
+ this.context.getNMTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -253,6 +256,8 @@ protected void serviceStart() throws Exception {
NodeId nodeId = NodeId.newInstance(connectAddress.getHostName(),
connectAddress.getPort());
((NodeManager.NMContext)context).setNodeId(nodeId);
+ this.context.getNMTokenSecretManager().setNodeId(nodeId);
+ this.context.getContainerTokenSecretManager().setNodeId(nodeId);
LOG.info("ContainerManager started at " + connectAddress);
super.serviceStart();
}
@@ -274,7 +279,7 @@ public void serviceStop() throws Exception {
}
// Get the remoteUGI corresponding to the api call.
- private UserGroupInformation getRemoteUgi()
+ protected UserGroupInformation getRemoteUgi()
throws YarnException {
UserGroupInformation remoteUgi;
try {
@@ -291,91 +296,67 @@ private UserGroupInformation getRemoteUgi()
// Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
// currently sets only the required id, but iterate through anyways just to
// be sure.
- private ContainerTokenIdentifier selectContainerTokenIdentifier(
+ @Private
+ @VisibleForTesting
+ protected NMTokenIdentifier selectNMTokenIdentifier(
UserGroupInformation remoteUgi) {
Set tokenIdentifiers = remoteUgi.getTokenIdentifiers();
- ContainerTokenIdentifier resultId = null;
+ NMTokenIdentifier resultId = null;
for (TokenIdentifier id : tokenIdentifiers) {
- if (id instanceof ContainerTokenIdentifier) {
- resultId = (ContainerTokenIdentifier) id;
+ if (id instanceof NMTokenIdentifier) {
+ resultId = (NMTokenIdentifier) id;
break;
}
}
return resultId;
}
- @Private
- @VisibleForTesting
- protected ContainerTokenIdentifier getContainerTokenIdentifier(
- UserGroupInformation remoteUgi,
- ContainerTokenIdentifier containerTokenIdentifier)
- throws YarnException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
- + remoteUgi.getTokenIdentifiers().size());
- }
- // Get the tokenId from the remote user ugi
- return selectContainerTokenIdentifier(remoteUgi);
- } else {
- return containerTokenIdentifier;
- }
- }
-
/**
- * Authorize the request.
- *
- * @param containerIDStr
- * of the container
- * @param launchContext
- * passed if verifying the startContainer, null otherwise.
- * @param remoteUgi
+ * @param containerTokenIdentifier
+ * of the container to be started
+ * @param ugi
* ugi corresponding to the remote end making the api-call
* @throws YarnException
*/
@Private
@VisibleForTesting
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext,
- UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
- throws YarnException {
+ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ ContainerId containerId = containerTokenIdentifier.getContainerID();
+ String containerIDStr = containerId.toString();
boolean unauthorized = false;
StringBuilder messageBuilder =
new StringBuilder("Unauthorized request to start container. ");
-
- if (!remoteUgi.getUserName().equals(containerIDStr)) {
+ if (!nmTokenIdentifier.getApplicationAttemptId().equals(
+ containerId.getApplicationAttemptId())) {
unauthorized = true;
- messageBuilder.append("\nExpected containerId: "
- + remoteUgi.getUserName() + " Found: " + containerIDStr);
- } else if (launchContext != null) {
- // Verify other things also for startContainer() request.
-
-
- if (tokenId == null) {
- unauthorized = true;
- messageBuilder
- .append("\nNo ContainerToken found for " + containerIDStr);
- } else {
-
- // Is the container being relaunched? Or RPC layer let startCall with
- // tokens generated off old-secret through?
- if (!this.context.getContainerTokenSecretManager()
- .isValidStartContainerRequest(tokenId.getContainerID())) {
- unauthorized = true;
- messageBuilder.append("\n Attempt to relaunch the same "
- + "container with id " + containerIDStr + ".");
- }
-
- // Ensure the token is not expired.
- // Token expiry is not checked for stopContainer/getContainerStatus
- if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
- unauthorized = true;
- messageBuilder.append("\nThis token is expired. current time is "
- + System.currentTimeMillis() + " found "
- + tokenId.getExpiryTimeStamp());
- }
- }
+ messageBuilder.append("\nNMToken for application attempt : ")
+ .append(nmTokenIdentifier.getApplicationAttemptId())
+ .append(" was used for starting container with container token")
+ .append(" issued for application attempt : ")
+ .append(containerId.getApplicationAttemptId());
+ } else if (!ugi.getUserName().equals(
+ nmTokenIdentifier.getApplicationAttemptId().toString())) {
+ unauthorized = true;
+ messageBuilder.append("\nExpected applicationAttemptId: ")
+ .append(ugi.getUserName()).append(" Found: ")
+ .append(nmTokenIdentifier.getApplicationAttemptId().toString());
+ } else if (!this.context.getContainerTokenSecretManager()
+ .isValidStartContainerRequest(containerId)) {
+ // Is the container being relaunched? Or RPC layer let startCall with
+ // tokens generated off old-secret through?
+ unauthorized = true;
+ messageBuilder.append("\n Attempt to relaunch the same ")
+ .append("container with id ").append(containerIDStr).append(".");
+ } else if (containerTokenIdentifier.getExpiryTimeStamp() < System
+ .currentTimeMillis()) {
+ // Ensure the token is not expired.
+ unauthorized = true;
+ messageBuilder.append("\nThis token is expired. current time is ")
+ .append(System.currentTimeMillis()).append(" found ")
+ .append(containerTokenIdentifier.getExpiryTimeStamp());
}
if (unauthorized) {
@@ -384,7 +365,7 @@ protected void authorizeRequest(String containerIDStr,
throw RPCUtil.getRemoteException(msg);
}
}
-
+
/**
* Start a container on this NodeManager.
*/
@@ -394,45 +375,134 @@ public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnException, IOException {
if (blockNewContainerRequests.get()) {
- throw new NMNotYetReadyException(
- "Rejecting new containers as NodeManager has not" +
- " yet connected with ResourceManager");
+ throw RPCUtil.getRemoteException(new NMNotYetReadyException(
+ "Rejecting new containers as NodeManager has not"
+ + " yet connected with ResourceManager"));
}
+ /*
+ * 1) It should save the NMToken into NMTokenSecretManager. This is done
+ * here instead of RPC layer because at the time of opening/authenticating
+ * the connection it doesn't know what all RPC calls user will make on it.
+ * Also new NMToken is issued only at startContainer (once it gets renewed).
+ *
+ * 2) It should validate containerToken. Need to check below things. a) It
+ * is signed by correct master key (part of retrieve password). b) It
+ * belongs to correct Node Manager (part of retrieve password). c) It has
+ * correct RMIdentifier. d) It is not expired.
+ */
+ // update NMToken
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
+
+ // Validate containerToken
+ ContainerTokenIdentifier containerTokenIdentifier =
+ verifyAndGetContainerTokenIdentifier(request.getContainerToken());
+
+ authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier,
+ remoteUgi);
+
+ if (containerTokenIdentifier.getRMIdentifer() != nodeStatusUpdater
+ .getRMIdentifier()) {
+ // Is the container coming from unknown RM
+ StringBuilder sb = new StringBuilder("\nContainer ");
+ sb.append(containerTokenIdentifier.getContainerID().toString())
+ .append(" rejected as it is allocated by a previous RM");
+ throw new InvalidContainerException(sb.toString());
+ }
+
+ updateNMTokenIdentifier(nmTokenIdentifier);
+
+ ContainerId containerId = containerTokenIdentifier.getContainerID();
+ String containerIdStr = containerId.toString();
+ String user = containerTokenIdentifier.getApplicationSubmitter();
+
+ LOG.info("Start request for " + containerIdStr + " by user " + user);
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
- org.apache.hadoop.yarn.api.records.Token token = request.getContainerToken();
- ContainerTokenIdentifier tokenIdentifier = null;
- try {
- tokenIdentifier = BuilderUtils.newContainerTokenIdentifier(token);
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
+ Credentials credentials = parseCredentials(launchContext);
+
+ Container container =
+ new ContainerImpl(getConfig(), this.dispatcher, launchContext,
+ credentials, metrics, containerTokenIdentifier);
+ ApplicationId applicationID =
+ containerId.getApplicationAttemptId().getApplicationId();
+ if (context.getContainers().putIfAbsent(containerId, container) != null) {
+ NMAuditLogger.logFailure(user, AuditConstants.START_CONTAINER,
+ "ContainerManagerImpl", "Container already running on this node!",
+ applicationID, containerId);
+ throw RPCUtil.getRemoteException("Container " + containerIdStr
+ + " already is running on this node!!");
}
- UserGroupInformation remoteUgi = getRemoteUgi();
- ContainerTokenIdentifier tokenId =
- getContainerTokenIdentifier(remoteUgi, tokenIdentifier);
+ // Create the application
+ Application application =
+ new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
+ credentials, context);
+ if (null == context.getApplications().putIfAbsent(applicationID,
+ application)) {
+ LOG.info("Creating a new application reference for app " + applicationID);
- ContainerId containerID = tokenId.getContainerID();
- String containerIDStr = containerID.toString();
+ dispatcher.getEventHandler().handle(
+ new ApplicationInitEvent(applicationID, container.getLaunchContext()
+ .getApplicationACLs()));
+ }
+
+ dispatcher.getEventHandler().handle(
+ new ApplicationContainerInitEvent(container));
- authorizeRequest(containerIDStr, launchContext, remoteUgi, tokenId);
+ this.context.getContainerTokenSecretManager().startContainerSuccessful(
+ containerTokenIdentifier);
+ NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER,
+ "ContainerManageImpl", applicationID, containerId);
+ StartContainerResponse response =
+ recordFactory.newRecordInstance(StartContainerResponse.class);
+ response.setAllServicesMetaData(auxiliaryServices.getMetaData());
+ // TODO launchedContainer misplaced -> doesn't necessarily mean a container
+ // launch. A finished Application will not launch containers.
+ metrics.launchedContainer();
+ metrics.allocateContainer(containerTokenIdentifier.getResource());
+ return response;
+ }
- // Is the container coming from unknown RM
- if (tokenId.getRMIdentifer() != nodeStatusUpdater
- .getRMIdentifier()) {
- String msg = "\nContainer "+ containerIDStr
- + " rejected as it is allocated by a previous RM";
- LOG.error(msg);
- throw new InvalidContainerException(msg);
+ protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
+ org.apache.hadoop.yarn.api.records.Token token) throws YarnException,
+ InvalidToken {
+ ContainerTokenIdentifier containerTokenIdentifier = null;
+ try {
+ containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(token);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
}
+ byte[] password =
+ context.getContainerTokenSecretManager().retrievePassword(
+ containerTokenIdentifier);
+ byte[] tokenPass = token.getPassword().array();
+ if (password == null || tokenPass == null
+ || !Arrays.equals(password, tokenPass)) {
+ throw new InvalidToken(
+ "Invalid container token used for starting container on : "
+ + context.getNodeId().toString());
+ }
+ return containerTokenIdentifier;
+ }
- LOG.info("Start request for " + containerIDStr + " by user "
- + tokenId.getApplicationSubmitter());
+ @Private
+ @VisibleForTesting
+ protected void updateNMTokenIdentifier(NMTokenIdentifier nmTokenIdentifier)
+ throws InvalidToken {
+ context.getNMTokenSecretManager().appAttemptStartContainer(
+ nmTokenIdentifier);
+ }
+ private Credentials parseCredentials(ContainerLaunchContext launchContext)
+ throws YarnException {
+ Credentials credentials = new Credentials();
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
- Credentials credentials = new Credentials();
+
if (tokens != null) {
DataInputByteBuffer buf = new DataInputByteBuffer();
tokens.rewind();
@@ -440,8 +510,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
try {
credentials.readTokenStorageStream(buf);
if (LOG.isDebugEnabled()) {
- for (Token extends TokenIdentifier> tk : credentials
- .getAllTokens()) {
+ for (Token extends TokenIdentifier> tk : credentials.getAllTokens()) {
LOG.debug(tk.getService() + " = " + tk.toString());
}
}
@@ -450,53 +519,7 @@ public StartContainerResponse startContainer(StartContainerRequest request)
}
}
// //////////// End of parsing credentials
- String user = tokenId.getApplicationSubmitter();
-
- Container container =
- new ContainerImpl(getConfig(), this.dispatcher, launchContext,
- credentials, metrics, tokenId);
- ApplicationId applicationID =
- containerID.getApplicationAttemptId().getApplicationId();
- if (context.getContainers().putIfAbsent(containerID, container) != null) {
- NMAuditLogger.logFailure(user,
- AuditConstants.START_CONTAINER, "ContainerManagerImpl",
- "Container already running on this node!",
- applicationID, containerID);
- throw RPCUtil.getRemoteException("Container " + containerIDStr
- + " already is running on this node!!");
- }
-
- // Create the application
- Application application =
- new ApplicationImpl(dispatcher, this.aclsManager,
- user, applicationID, credentials,
- context);
- if (null ==
- context.getApplications().putIfAbsent(applicationID, application)) {
- LOG.info("Creating a new application reference for app "
- + applicationID);
- dispatcher.getEventHandler().handle(
- new ApplicationInitEvent(applicationID, container
- .getLaunchContext().getApplicationACLs()));
- }
-
- // TODO: Validate the request
- dispatcher.getEventHandler().handle(
- new ApplicationContainerInitEvent(container));
-
- this.context.getContainerTokenSecretManager().startContainerSuccessful(
- tokenId);
- NMAuditLogger.logSuccess(user,
- AuditConstants.START_CONTAINER, "ContainerManageImpl",
- applicationID, containerID);
-
- StartContainerResponse response =
- StartContainerResponse.newInstance(auxiliaryServices.getMetaData());
- // TODO launchedContainer misplaced -> doesn't necessarily mean a container
- // launch. A finished Application will not launch containers.
- metrics.launchedContainer();
- metrics.allocateContainer(tokenId.getResource());
- return response;
+ return credentials;
}
/**
@@ -509,34 +532,20 @@ public StopContainerResponse stopContainer(StopContainerRequest request)
ContainerId containerID = request.getContainerId();
String containerIDStr = containerID.toString();
-
- // TODO: Only the container's owner can kill containers today.
-
- UserGroupInformation remoteUgi = getRemoteUgi();
Container container = this.context.getContainers().get(containerID);
+ LOG.info("Getting container-status for " + containerIDStr);
+ authorizeGetAndStopContainerRequest(containerID, container, true);
+
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
- if (container == null) {
- LOG.warn("Trying to stop unknown container " + containerID);
- NMAuditLogger.logFailure("UnknownUser",
- AuditConstants.STOP_CONTAINER, "ContainerManagerImpl",
- "Trying to stop unknown container!",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
- return response; // Return immediately.
- }
- authorizeRequest(containerIDStr, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
-
dispatcher.getEventHandler().handle(
- new ContainerKillEvent(containerID,
- "Container killed by the ApplicationMaster."));
-
- NMAuditLogger.logSuccess(container.getUser(),
- AuditConstants.STOP_CONTAINER, "ContainerManageImpl",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
+ new ContainerKillEvent(containerID,
+ "Container killed by the ApplicationMaster."));
+
+ NMAuditLogger.logSuccess(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
+ .getApplicationAttemptId().getApplicationId(), containerID);
// TODO: Move this code to appropriate place once kill_container is
// implemented.
@@ -547,23 +556,14 @@ public StopContainerResponse stopContainer(StopContainerRequest request)
@Override
public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException,
- IOException {
+ GetContainerStatusRequest request) throws YarnException, IOException {
ContainerId containerID = request.getContainerId();
String containerIDStr = containerID.toString();
+ Container container = this.context.getContainers().get(containerID);
- // TODO: Only the container's owner can get containers' status today.
-
- UserGroupInformation remoteUgi = getRemoteUgi();
LOG.info("Getting container-status for " + containerIDStr);
- Container container = this.context.getContainers().get(containerID);
- if (container == null) {
- throw RPCUtil.getRemoteException("Container " + containerIDStr
- + " is not handled by this NodeManager");
- }
- authorizeRequest(containerIDStr, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
+ authorizeGetAndStopContainerRequest(containerID, container, false);
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
@@ -573,6 +573,48 @@ public GetContainerStatusResponse getContainerStatus(
return response;
}
+ @Private
+ @VisibleForTesting
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
+
+ /*
+ * For get/stop container status; we need to verify that 1) User (NMToken)
+ * application attempt only has started container. 2) Requested containerId
+ * belongs to the same application attempt (NMToken) which was used. (Note:-
+ * This will prevent user in knowing another application's containers).
+ */
+
+ if ((!identifier.getApplicationAttemptId().equals(
+ containerId.getApplicationAttemptId()))
+ || (container != null && !identifier.getApplicationAttemptId().equals(
+ container.getContainerId().getApplicationAttemptId()))) {
+ if (stopRequest) {
+ LOG.warn(identifier.getApplicationAttemptId()
+ + " attempted to stop non-application container : "
+ + container.getContainerId().toString());
+ NMAuditLogger.logFailure("UnknownUser", AuditConstants.STOP_CONTAINER,
+ "ContainerManagerImpl", "Trying to stop unknown container!",
+ identifier.getApplicationAttemptId().getApplicationId(),
+ container.getContainerId());
+ } else {
+ LOG.warn(identifier.getApplicationAttemptId()
+ + " attempted to get get status for non-application container : "
+ + container.getContainerId().toString());
+ }
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is not started by this application attempt.");
+ }
+
+ if (container == null) {
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is not handled by this NodeManager");
+ }
+ }
+
class ContainerEventDispatcher implements EventHandler {
@Override
public void handle(ContainerEvent event) {
@@ -643,9 +685,19 @@ public void setBlockNewContainerRequests(boolean blockNewContainerRequests) {
this.blockNewContainerRequests.set(blockNewContainerRequests);
}
+ @Private
+ @VisibleForTesting
+ public boolean getBlockNewContainerRequestsStatus() {
+ return this.blockNewContainerRequests.get();
+ }
+
@Override
public void stateChanged(Service service) {
// TODO Auto-generated method stub
}
+
+ public Context getContext() {
+ return this.context;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java
index 3f77059..5eff43b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java
@@ -18,26 +18,32 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security;
+import javax.crypto.SecretKey;
+
import org.apache.hadoop.security.token.SecretManager;
public class LocalizerTokenSecretManager extends
SecretManager {
+ private final SecretKey secretKey;
+
+ public LocalizerTokenSecretManager() {
+ this.secretKey = generateSecret();
+ }
+
@Override
protected byte[] createPassword(LocalizerTokenIdentifier identifier) {
- return "testing".getBytes();
+ return createPassword(identifier.getBytes(), secretKey);
}
@Override
public byte[] retrievePassword(LocalizerTokenIdentifier identifier)
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
- // TODO Auto-generated method stub
- return "testing".getBytes();
+ return createPassword(identifier.getBytes(), secretKey);
}
@Override
public LocalizerTokenIdentifier createIdentifier() {
- // TODO Auto-generated method stub
return new LocalizerTokenIdentifier();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
index c5e00f2..bc349f4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
@@ -30,13 +30,12 @@
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* The NM maintains only two master-keys. The current key that RM knows and the
* key from the previous rolling-interval.
@@ -51,6 +50,7 @@
private MasterKeyData previousMasterKey;
private final Map> oldMasterKeys;
+ private String nodeHostAddr;
public NMContainerTokenSecretManager(Configuration conf) {
super(conf);
@@ -122,6 +122,15 @@ public synchronized void setMasterKey(MasterKey masterKeyRecord) {
masterKeyToUse = this.oldMasterKeys.get(appId).get(containerId);
}
+ if (nodeHostAddr != null
+ && !identifier.getNmHostAddress().equals(nodeHostAddr)) {
+ // Valid container token used for incorrect node.
+ throw new SecretManager.InvalidToken("Given Container "
+ + identifier.getContainerID().toString()
+ + " identifier is not valid for current Node manager. Expected : "
+ + nodeHostAddr + " Found : " + identifier.getNmHostAddress());
+ }
+
if (masterKeyToUse != null) {
return retrievePasswordInternal(identifier, masterKeyToUse);
}
@@ -186,4 +195,9 @@ private synchronized void addKeyForContainerId(ContainerId containerId,
public synchronized void appFinished(ApplicationId appId) {
this.oldMasterKeys.remove(appId);
}
+
+ public synchronized void setNodeId(NodeId nodeId) {
+ nodeHostAddr = nodeId.toString();
+ LOG.info("Updating node address : " + nodeHostAddr);
+ }
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
index 28c40e6..1ecd4da 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
@@ -18,18 +18,24 @@
package org.apache.hadoop.yarn.server.nodemanager.security;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
+import com.google.common.annotations.VisibleForTesting;
+
public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
private static final Log LOG = LogFactory
@@ -38,10 +44,15 @@
private MasterKeyData previousMasterKey;
private final Map oldMasterKeys;
+ private final Map> appToAppAttemptMap;
+ private NodeId nodeId;
+
public NMTokenSecretManagerInNM() {
this.oldMasterKeys =
new HashMap();
+ appToAppAttemptMap =
+ new HashMap>();
}
/**
@@ -69,46 +80,117 @@ public synchronized void setMasterKey(MasterKey masterKey) {
}
/**
- * This method will be used to verify NMTokens generated by different
- * master keys.
+ * This method will be used to verify NMTokens generated by different master
+ * keys.
*/
@Override
- public synchronized byte[] retrievePassword(
- NMTokenIdentifier identifier) throws InvalidToken {
- int keyId = identifier.getMastKeyId();
+ public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
+ throws InvalidToken {
+ int keyId = identifier.getKeyId();
ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();
-
+
/*
- * MasterKey used for retrieving password will be as follows.
- * 1) By default older saved master key will be used.
- * 2) If identifier's master key id matches that of previous master key
- * id then previous key will be used.
- * 3) If identifier's master key id matches that of current master key
- * id then current key will be used.
+ * MasterKey used for retrieving password will be as follows. 1) By default
+ * older saved master key will be used. 2) If identifier's master key id
+ * matches that of previous master key id then previous key will be used. 3)
+ * If identifier's master key id matches that of current master key id then
+ * current key will be used.
*/
MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
MasterKeyData masterKeyToUse = oldMasterKey;
if (previousMasterKey != null
&& keyId == previousMasterKey.getMasterKey().getKeyId()) {
masterKeyToUse = previousMasterKey;
- } else if ( keyId == currentMasterKey.getMasterKey().getKeyId()) {
+ } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
masterKeyToUse = currentMasterKey;
}
+ if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
+ throw new InvalidToken("Given NMToken for application : "
+ + appAttemptId.toString() + " is not valid for current node manager."
+ + "expected : " + nodeId.toString() + " found : "
+ + identifier.getNodeId().toString());
+ }
+
if (masterKeyToUse != null) {
byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
- if (masterKeyToUse.getMasterKey().getKeyId() != oldMasterKey
- .getMasterKey().getKeyId()) {
- oldMasterKeys.put(appAttemptId, masterKeyToUse);
- }
+ LOG.debug("NMToken password retrieved successfully!!");
return password;
}
-
+
throw new InvalidToken("Given NMToken for application : "
+ appAttemptId.toString() + " seems to have been generated illegally.");
}
+
+ public synchronized void appFinished(ApplicationId appId) {
+ List appAttemptList = appToAppAttemptMap.get(appId);
+ if (appAttemptList != null) {
+ LOG.debug("Removing application attempts NMToken keys for application "
+ + appId);
+ for (ApplicationAttemptId appAttemptId : appAttemptList) {
+ this.oldMasterKeys.remove(appAttemptId);
+ }
+ appToAppAttemptMap.remove(appId);
+ } else {
+ LOG.error("No application Attempt for application : " + appId
+ + " started on this NM.");
+ }
+ }
+
+ /**
+ * This will be called by startContainer. It will add the master key into
+ * the cache used for starting this container. This should be called before
+ * validating the startContainer request.
+ */
+ public synchronized void appAttemptStartContainer(
+ NMTokenIdentifier identifier)
+ throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
+ ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();
+ if (!appToAppAttemptMap.containsKey(appAttemptId.getApplicationId())) {
+ // First application attempt for the given application
+ appToAppAttemptMap.put(appAttemptId.getApplicationId(),
+ new ArrayList());
+ }
+ MasterKeyData oldKey = oldMasterKeys.get(appAttemptId);
+
+ if (oldKey == null) {
+ // This is a new application attempt.
+ appToAppAttemptMap.get(appAttemptId.getApplicationId()).add(appAttemptId);
+ }
+ if (oldKey == null
+ || oldKey.getMasterKey().getKeyId() != identifier.getKeyId()) {
+ // Update key only if it is modified.
+ LOG.debug("NMToken key updated for application attempt : "
+ + identifier.getApplicationAttemptId().toString());
+ if (identifier.getKeyId() == currentMasterKey.getMasterKey()
+ .getKeyId()) {
+ oldMasterKeys.put(appAttemptId, currentMasterKey);
+ } else if (previousMasterKey != null
+ && identifier.getKeyId() == previousMasterKey.getMasterKey()
+ .getKeyId()) {
+ oldMasterKeys.put(appAttemptId, previousMasterKey);
+ } else {
+ throw new InvalidToken(
+ "Older NMToken should not be used while starting the container.");
+ }
+ }
+ }
+
+ public void setNodeId(NodeId nodeId) {
+ LOG.debug("updating nodeId : " + nodeId);
+ this.nodeId = nodeId;
+ }
- public synchronized void appFinished(ApplicationAttemptId appAttemptId) {
- this.oldMasterKeys.remove(appAttemptId);
+ @Private
+ @VisibleForTesting
+ public synchronized boolean
+ isAppAttemptNMTokenKeyPresent(ApplicationAttemptId appAttemptId) {
+ return oldMasterKeys.containsKey(appAttemptId);
+ }
+
+ @Private
+ @VisibleForTesting
+ public NodeId getNodeId() {
+ return this.nodeId;
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index d83f9b6..76ef074 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -27,10 +27,12 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -126,6 +128,19 @@ public void handle(LocalizationEvent event) {
}
@Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, getContext()
+ .getNodeId(), "testuser", getContext().getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ @Override
@SuppressWarnings("unchecked")
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
@@ -179,17 +194,16 @@ public void setBlockNewContainerRequests(boolean blockNewContainerRequests) {
}
@Override
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext,
- UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
- throws YarnException {
- // do Nothing
+ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ // do nothing
}
-
+
@Override
- protected ContainerTokenIdentifier
- getContainerTokenIdentifier(UserGroupInformation remoteUgi,
- ContainerTokenIdentifier containerTokenId) throws YarnException {
- return containerTokenId;
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+ // do nothing
}
+
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index fe2655d..13dc099 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -74,7 +74,7 @@ public void tearDown() throws IOException, InterruptedException {
}
@Override
- public void testContainerSetup() throws IOException, InterruptedException,
+ public void testContainerSetup() throws Exception, InterruptedException,
YarnException {
// Don't run the test if the binary is not available.
if (!shouldRunTest()) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index 9946704..668b85b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -133,18 +134,13 @@ public long getRMIdentifier() {
ApplicationAttemptId.newInstance(applicationId, 0);
ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0);
- Resource r = BuilderUtils.newResource(1024, 1);
String user = "testing";
- String host = "127.0.0.1";
- int port = 1234;
- Token containerToken =
- BuilderUtils.newContainerToken(cID, host, port, user, r,
- System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
- SIMULATED_RM_IDENTIFIER);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
- request.setContainerToken(containerToken);
+ request.setContainerToken(TestContainerManager.createContainerToken(cID,
+ SIMULATED_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index cda47b2..3b88b03 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -50,17 +49,16 @@
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -131,24 +129,23 @@ public void testClearLocalDirWhenNodeReboot() throws IOException,
containerLaunchContext.setLocalResources(localResources);
List commands = new ArrayList();
containerLaunchContext.setCommands(commands);
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemory(1024);
- NodeId nodeId = BuilderUtils.newNodeId("127.0.0.1", 12345);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
- user, resource, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), 0);
final StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ NodeId nodeId = nm.getNMContext().getNodeId();
+ startRequest.setContainerToken(TestContainerManager.createContainerToken(
+ cId, 0, nodeId, destinationFile, nm.getNMContext()
+ .getContainerTokenSecretManager()));
final UserGroupInformation currentUser = UserGroupInformation
- .createRemoteUser(cId.toString());
+ .createRemoteUser(cId.getApplicationAttemptId().toString());
+ NMTokenIdentifier nmIdentifier =
+ new NMTokenIdentifier(cId.getApplicationAttemptId(), nodeId, user, 123);
+ currentUser.addTokenIdentifier(nmIdentifier);
currentUser.doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws YarnException, IOException {
- containerManager.startContainer(startRequest);
+ nm.getContainerManager().startContainer(startRequest);
return null;
}
});
@@ -208,8 +205,6 @@ public Void run() throws YarnException, IOException {
ContainerLocalizer.FILECACHE) == 0 && numOfLocalDirs(nmLocalDir
.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR)
== 0);
- verify(delService, times(1)).delete(eq(user),
- argThat(new PathInclude(user)));
verify(delService, times(1)).delete(
(String) isNull(),
argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
index ad33fa6..fe3c5da 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
@@ -52,16 +52,17 @@
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
@@ -161,7 +162,7 @@ public static void startContainer(NodeManager nm, ContainerId cId,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
+ NodeId nodeId = BuilderUtils.newNodeId("localhost", 12345);
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -180,17 +181,22 @@ public static void startContainer(NodeManager nm, ContainerId cId,
containerLaunchContext.setLocalResources(localResources);
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource resource = BuilderUtils.newResource(1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
- user, resource, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), 0);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest
+ .setContainerToken(TestContainerManager.createContainerToken(cId, 0,
+ nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
+ final InetSocketAddress containerManagerBindAddress =
+ NetUtils.createSocketAddrForHost("127.0.0.1", 12345);
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(cId.toString());
+ org.apache.hadoop.security.token.Token nmToken =
+ ConverterUtils.convertFromYarn(
+ nm.getNMContext().getNMTokenSecretManager()
+ .createNMToken(cId.getApplicationAttemptId(), nodeId, user),
+ containerManagerBindAddress);
+ currentUser.addToken(nmToken);
ContainerManagementProtocol containerManager =
currentUser.doAs(new PrivilegedAction() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 843fd3c..e17131f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -201,7 +201,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
String user = "testUser";
ContainerTokenIdentifier containerToken =
BuilderUtils.newContainerTokenIdentifier(BuilderUtils
- .newContainerToken(firstContainerID, "127.0.0.1", 1234, user,
+ .newContainerToken(firstContainerID, "localhost", 1234, user,
resource, currentTime + 10000, 123, "password".getBytes(),
currentTime));
Container container =
@@ -232,7 +232,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
Resource resource = BuilderUtils.newResource(3, 1);
ContainerTokenIdentifier containerToken =
BuilderUtils.newContainerTokenIdentifier(BuilderUtils
- .newContainerToken(secondContainerID, "127.0.0.1", 1234, user,
+ .newContainerToken(secondContainerID, "localhost", 1234, user,
resource, currentTime + 10000, 123,
"password".getBytes(), currentTime));
Container container =
@@ -1168,8 +1168,8 @@ private void verifyNodeStartFailure(String errMessage) throws Exception {
private YarnConfiguration createNMConfig() {
YarnConfiguration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
- conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
- conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+ conf.set(YarnConfiguration.NM_ADDRESS, "localhost:12345");
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "localhost:12346");
conf.set(YarnConfiguration.NM_LOG_DIRS, new Path(basedir, "logs").toUri()
.getPath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, new Path(basedir,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 6c9198c..2a157d8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -30,21 +30,20 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -58,6 +57,7 @@
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -150,7 +150,7 @@ public void setup() throws IOException {
LOG.info("Created localDir in " + localDir.getAbsolutePath());
LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
- String bindAddress = "0.0.0.0:5555";
+ String bindAddress = "127.0.0.1:12345";
conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
@@ -173,6 +173,7 @@ public void setup() throws IOException {
protected ContainerManagerImpl
createContainerManager(DeletionService delSrvc) {
+
return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
metrics, new ApplicationACLsManager(conf), dirsHandler) {
@Override
@@ -182,11 +183,24 @@ public void setup() throws IOException {
}
@Override
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext, UserGroupInformation remoteUgi,
- ContainerTokenIdentifier tokenId) throws YarnException {
- // do nothing
- }
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+ // do nothing
+ }
+
+ @Override
+ protected void authorizeStartRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ // do nothing
+ }
+
+ @Override
+ protected void updateNMTokenIdentifier(
+ NMTokenIdentifier nmTokenIdentifier) throws InvalidToken {
+ // Do nothing
+ }
};
}
@@ -242,7 +256,7 @@ static void waitForApplicationState(ContainerManagerImpl containerManager,
throws InterruptedException {
// Wait for app-finish
Application app =
- containerManager.context.getApplications().get(appID);
+ containerManager.getContext().getApplications().get(appID);
int timeout = 0;
while (!(app.getApplicationState().equals(finalState))
&& timeout++ < 15) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 5fb8812..a66be90 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -47,10 +48,13 @@
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
@@ -59,8 +63,11 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Before;
import org.junit.Test;
public class TestContainerManager extends BaseContainerManagerTest {
@@ -72,6 +79,12 @@ public TestContainerManager() throws UnsupportedFileSystemException {
static {
LOG = LogFactory.getLog(TestContainerManager.class);
}
+
+ @Override
+ @Before
+ public void setup() throws IOException {
+ super.setup();
+ }
private ContainerId createContainerId() {
ApplicationId appId = ApplicationId.newInstance(0, 0);
@@ -81,6 +94,32 @@ private ContainerId createContainerId() {
return containerId;
}
+ @Override
+ protected ContainerManagerImpl
+ createContainerManager(DeletionService delSrvc) {
+ return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler) {
+ @Override
+ public void
+ setBlockNewContainerRequests(boolean blockNewContainerRequests) {
+ // do nothing
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+ };
+ }
+
@Test
public void testContainerManagerInitialization() throws IOException {
@@ -101,8 +140,7 @@ public void testContainerManagerInitialization() throws IOException {
}
@Test
- public void testContainerSetup() throws IOException, InterruptedException,
- YarnException {
+ public void testContainerSetup() throws Exception {
containerManager.start();
@@ -134,16 +172,12 @@ public void testContainerSetup() throws IOException, InterruptedException,
new HashMap();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Resource r = BuilderUtils.newResource(512, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
@@ -227,16 +261,12 @@ public void testContainerLaunchAndStop() throws IOException,
containerLaunchContext.setLocalResources(localResources);
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -335,15 +365,12 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException,
containerLaunchContext.setLocalResources(localResources);
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
+
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -423,16 +450,10 @@ public void testLocalFilesCleanup() throws InterruptedException,
new HashMap();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
-
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
- request.setContainerToken(containerToken);
+ request.setContainerToken(createContainerToken(cId, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user, context.getContainerTokenSecretManager()));
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -503,24 +524,19 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- String host = "127.0.0.1";
- int port = 1234;
ContainerId cId1 = createContainerId();
ContainerId cId2 = createContainerId();
containerLaunchContext
.setLocalResources(new HashMap());
- Resource mockResource = BuilderUtils.newResource(1024, 1);
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest1.setContainerLaunchContext(containerLaunchContext);
- Token containerToken1 =
- BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
- System.currentTimeMillis() + 10000, 123, "password".getBytes(),
- (long) ResourceManagerConstants.RM_INVALID_IDENTIFIER);
- startRequest1.setContainerToken(containerToken1);
+ startRequest1.setContainerToken(createContainerToken(cId1,
+ ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
boolean catchException = false;
try {
containerManager.startContainer(startRequest1);
@@ -528,8 +544,8 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
catchException = true;
Assert.assertTrue(e.getMessage().contains(
"Container " + cId1 + " rejected as it is allocated by a previous RM"));
- Assert.assertEquals(InvalidContainerException.class.getName(), e
- .getClass().getName());
+ Assert.assertTrue(e.getClass().getName()
+ .equalsIgnoreCase(InvalidContainerException.class.getName()));
}
// Verify that startContainer fail because of invalid container request
@@ -539,11 +555,9 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
- Token containerToken2 =
- BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
- System.currentTimeMillis() + 10000, 123, "password".getBytes(),
- super.DUMMY_RM_IDENTIFIER);
- startRequest2.setContainerToken(containerToken2);
+ startRequest2.setContainerToken(createContainerToken(cId2,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
boolean noException = true;
try {
containerManager.startContainer(startRequest2);
@@ -553,4 +567,20 @@ public void testContainerLaunchFromPreviousRM() throws IOException,
// Verify that startContainer get no YarnException
Assert.assertTrue(noException);
}
+
+ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
+ NodeId nodeId, String user,
+ NMContainerTokenSecretManager containerTokenSecretManager)
+ throws IOException {
+ Resource r = BuilderUtils.newResource(1024, 1);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
+ System.currentTimeMillis() + 100000L, 123, rmIdentifier);
+ Token containerToken =
+ BuilderUtils
+ .newContainerToken(nodeId, containerTokenSecretManager
+ .retrievePassword(containerTokenIdentifier),
+ containerTokenIdentifier);
+ return containerToken;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index b08fe27..231e2fa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -56,6 +57,7 @@
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -229,14 +231,9 @@ public void testContainerEnvVariables() throws Exception {
// set up the rest of the container
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(1024, 1);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 1234,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId));
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -378,12 +375,9 @@ public void testDelayedKill() throws Exception {
// set up the rest of the container
List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
+ Token containerToken = createContainerToken(cId);
+ StartContainerRequest startRequest =
+ recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
@@ -441,4 +435,17 @@ public void testDelayedKill() throws Exception {
}
}
+ protected Token createContainerToken(ContainerId cId) throws InvalidToken {
+ Resource r = BuilderUtils.newResource(1024, 1);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
+ r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER);
+ Token containerToken =
+ BuilderUtils.newContainerToken(
+ context.getNodeId(),
+ context.getContainerTokenSecretManager().retrievePassword(
+ containerTokenIdentifier), containerTokenIdentifier);
+ return containerToken;
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 58a96c9..152b988 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -23,9 +23,9 @@
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyMap;
import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
@@ -73,8 +73,6 @@
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -94,6 +92,7 @@
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
@@ -810,15 +809,12 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
- System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
- super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(TestContainerManager.createContainerToken(
+ cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
this.containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(this.containerManager,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml
new file mode 100644
index 0000000..f0d3085
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml
@@ -0,0 +1,25 @@
+
+
+
+
+
+
+
+
+ hadoop.security.token.service.use_ip
+ false
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 7cbfc1c..f6b4523 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -380,7 +380,7 @@ public AllocateResponse allocate(AllocateRequest request)
// Adding NMTokens for allocated containers.
if (!allocation.getContainers().isEmpty()) {
allocateResponse.setNMTokens(rmContext.getNMTokenSecretManager()
- .getNMTokens(app.getUser(), appAttemptId,
+ .createAndGetNMTokens(app.getUser(), appAttemptId,
allocation.getContainers()));
}
return allocateResponse;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 673c7e4..51a0400 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -131,21 +131,30 @@ protected ContainerManagementProtocol getContainerMgrProxy(
final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again.
- UserGroupInformation currentUser = UserGroupInformation
- .createRemoteUser(containerId.toString());
- if (UserGroupInformation.isSecurityEnabled()) {
- Token token =
- ConverterUtils.convertFromYarn(masterContainer
- .getContainerToken(), containerManagerBindAddress);
- currentUser.addToken(token);
- }
- return currentUser.doAs(new PrivilegedAction() {
- @Override
- public ContainerManagementProtocol run() {
- return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class,
- containerManagerBindAddress, conf);
- }
- });
+ UserGroupInformation currentUser =
+ UserGroupInformation.createRemoteUser(containerId
+ .getApplicationAttemptId().toString());
+
+ String user =
+ rmContext.getRMApps()
+ .get(containerId.getApplicationAttemptId().getApplicationId())
+ .getUser();
+ org.apache.hadoop.yarn.api.records.Token token =
+ rmContext.getNMTokenSecretManager().createNMToken(
+ containerId.getApplicationAttemptId(), node, user);
+ currentUser.addToken(ConverterUtils.convertFromYarn(token,
+ containerManagerBindAddress));
+
+ return currentUser
+ .doAs(new PrivilegedAction() {
+
+ @Override
+ public ContainerManagementProtocol run() {
+ return (ContainerManagementProtocol) rpc.getProxy(
+ ContainerManagementProtocol.class,
+ containerManagerBindAddress, conf);
+ }
+ });
}
private ContainerLaunchContext createAMContainerLaunchContext(
@@ -234,7 +243,13 @@ public void run() {
} catch(IOException ie) {
LOG.info("Error cleaning master ", ie);
} catch (YarnException e) {
- LOG.info("Error cleaning master ", e);
+ StringBuilder sb = new StringBuilder("Container ");
+ sb.append(masterContainer.getId().toString());
+ sb.append(" is not handled by this NodeManager");
+ if (!e.getMessage().contains(sb.toString())) {
+ // Ignoring if container is already killed by Node Manager.
+ LOG.info("Error cleaning master ", e);
+ }
}
break;
default:
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
index 3931c6c..ab31eaf 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
@@ -31,16 +30,12 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
@@ -183,7 +178,7 @@ public void run() {
}
}
- public List getNMTokens(String applicationSubmitter,
+ public List createAndGetNMTokens(String applicationSubmitter,
ApplicationAttemptId appAttemptId, List containers) {
try {
this.readLock.lock();
@@ -193,12 +188,14 @@ public void run() {
for (Container container : containers) {
if (!nodeSet.contains(container.getNodeId())) {
LOG.debug("Sending NMToken for nodeId : "
- + container.getNodeId().toString());
+ + container.getNodeId().toString()
+ + " for application attempt : " + appAttemptId.toString());
Token token = createNMToken(appAttemptId, container.getNodeId(),
applicationSubmitter);
NMToken nmToken =
NMToken.newInstance(container.getNodeId(), token);
nmTokens.add(nmToken);
+ // This will update the nmToken set.
nodeSet.add(container.getNodeId());
}
}
@@ -273,38 +270,4 @@ public void removeNodeKey(NodeId nodeId) {
this.writeLock.unlock();
}
}
-
- public static Token newNMToken(byte[] password,
- NMTokenIdentifier identifier) {
- NodeId nodeId = identifier.getNodeId();
- // RPC layer client expects ip:port as service for tokens
- InetSocketAddress addr =
- NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
- Token nmToken =
- Token.newInstance(identifier.getBytes(),
- NMTokenIdentifier.KIND.toString(), password, SecurityUtil
- .buildTokenService(addr).toString());
- return nmToken;
- }
-
- /**
- * Helper function for creating NMTokens.
- */
- public Token createNMToken(ApplicationAttemptId applicationAttemptId,
- NodeId nodeId, String applicationSubmitter) {
- byte[] password;
- NMTokenIdentifier identifier;
-
- this.readLock.lock();
- try {
- identifier =
- new NMTokenIdentifier(applicationAttemptId, nodeId,
- applicationSubmitter, this.currentMasterKey.getMasterKey()
- .getKeyId());
- password = this.createPassword(identifier);
- } finally {
- this.readLock.unlock();
- }
- return newNMToken(password, identifier);
- }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index c1d3f92..522debb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -59,9 +59,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -310,6 +308,7 @@ protected ResourceTrackerService createResourceTrackerService() {
Configuration conf = new Configuration();
containerTokenSecretManager.rollMasterKey();
+ nmTokenSecretManager.rollMasterKey();
return new ResourceTrackerService(getRMContext(), nodesListManager,
this.nmLivelinessMonitor, containerTokenSecretManager,
nmTokenSecretManager) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index a4ccb43..b78a1e5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -22,14 +22,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
import junit.framework.Assert;
@@ -37,45 +30,30 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -111,15 +89,15 @@ private void testContainerManager() throws Exception {
yarnCluster.init(conf);
yarnCluster.start();
- // Testing for authenticated user
- testAuthenticatedUser();
+ // TestNMTokens.
+ testNMTokens(conf);
- // Testing for malicious user
- testMaliceUser();
-
- // Testing for usage of expired tokens
- testExpiredTokens();
+ // Testing for container token tampering
+ testContainerToken(conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
} finally {
if (yarnCluster != null) {
yarnCluster.stop();
@@ -128,411 +106,330 @@ private void testContainerManager() throws Exception {
}
}
- private void testAuthenticatedUser() throws IOException,
- InterruptedException, YarnException {
-
- LOG.info("Running test for authenticated user");
-
- ResourceManager resourceManager = yarnCluster.getResourceManager();
-
- final YarnRPC yarnRPC = YarnRPC.create(conf);
-
- // Submit an application
- ApplicationId appID = resourceManager.getClientRMService()
- .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
- .getApplicationId();
- ApplicationMasterProtocol scheduler = submitAndRegisterApplication(resourceManager,
- yarnRPC, appID);
-
- // Now request a container.
- final Container allocatedContainer = requestAndGetContainer(scheduler,
- appID);
-
- // Now talk to the NM for launching the container.
- final ContainerId containerID = allocatedContainer.getId();
- UserGroupInformation authenticatedUser = UserGroupInformation
- .createRemoteUser(containerID.toString());
- org.apache.hadoop.yarn.api.records.Token containerToken =
- allocatedContainer.getContainerToken();
- Token token = new Token(
- containerToken.getIdentifier().array(), containerToken.getPassword()
- .array(), new Text(containerToken.getKind()), new Text(
- containerToken.getService()));
- authenticatedUser.addToken(token);
- authenticatedUser.doAs(new PrivilegedExceptionAction() {
- @Override
- public Void run() throws Exception {
- ContainerManagementProtocol client = (ContainerManagementProtocol) yarnRPC.getProxy(
- ContainerManagementProtocol.class, NetUtils
- .createSocketAddr(allocatedContainer.getNodeId().toString()),
- conf);
- LOG.info("Going to make a legal stopContainer() request");
- StopContainerRequest request = recordFactory
- .newRecordInstance(StopContainerRequest.class);
- request.setContainerId(containerID);
- client.stopContainer(request);
- return null;
- }
- });
+ private void testNMTokens(Configuration conf) throws Exception {
+ NMTokenSecretManagerInRM nmTokenSecretManagerRM =
+ yarnCluster.getResourceManager().getRMContext()
+ .getNMTokenSecretManager();
+ NMTokenSecretManagerInNM nmTokenSecretManagerNM =
+ yarnCluster.getNodeManager(0).getNMContext().getNMTokenSecretManager();
+ RMContainerTokenSecretManager containerTokenSecretManager =
+ yarnCluster.getResourceManager().getRMContainerTokenSecretManager();
+
+ NodeManager nm = yarnCluster.getNodeManager(0);
+
+ waitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM, nm);
+
+ // Both id should be equal.
+ Assert.assertEquals(nmTokenSecretManagerNM.getCurrentKey().getKeyId(),
+ nmTokenSecretManagerRM.getCurrentKey().getKeyId());
+
+ /*
+ * Below cases should be tested.
+ * 1) If Invalid NMToken is used then it should be rejected.
+ * 2) If valid NMToken but belonging to another Node is used then that
+ * too should be rejected.
+ * 3) NMToken for say appAttempt-1 is used for starting/stopping/retrieving
+ * status for container with containerId for say appAttempt-2 should
+ * be rejected.
+ * 4) After start container call is successful nmtoken should have been
+ * saved in NMTokenSecretManagerInNM.
+ * 5) If start container call was successful (no matter if container is
+ * still running or not), appAttempt->NMToken should be present in
+ * NMTokenSecretManagerInNM's cache. Any future getContainerStatus call
+ * for containerId belonging to that application attempt using
+ * applicationAttempt's older nmToken should not get any invalid
+ * nmToken error. (This can be best tested if we roll over NMToken
+ * master key twice).
+ */
+ YarnRPC rpc = YarnRPC.create(conf);
+ String user = "test";
+ Resource r = Resource.newInstance(1024, 1);
+
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId validAppAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ApplicationAttemptId invalidAppAttemptId =
+ ApplicationAttemptId.newInstance(appId, 2);
+
+ ContainerId validContainerId =
+ ContainerId.newInstance(validAppAttemptId, 0);
+
+ NodeId validNode = yarnCluster.getNodeManager(0).getNMContext().getNodeId();
+ NodeId invalidNode = NodeId.newInstance("InvalidHost", 1234);
- KillApplicationRequest request = Records
- .newRecord(KillApplicationRequest.class);
- request.setApplicationId(appID);
- resourceManager.getClientRMService().forceKillApplication(request);
+
+ org.apache.hadoop.yarn.api.records.Token validNMToken =
+ nmTokenSecretManagerRM.createNMToken(validAppAttemptId, validNode, user);
+
+ org.apache.hadoop.yarn.api.records.Token validContainerToken =
+ containerTokenSecretManager.createContainerToken(validContainerId,
+ validNode, user, r);
+
+ StringBuilder sb;
+ // testInvalidNMToken ... creating NMToken using different secret manager.
+
+ NMTokenSecretManagerInRM tempManager = new NMTokenSecretManagerInRM(conf);
+ tempManager.rollMasterKey();
+ do {
+ tempManager.rollMasterKey();
+ tempManager.activateNextMasterKey();
+ // Making sure key id is different.
+ } while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM
+ .getCurrentKey().getKeyId());
+
+ org.apache.hadoop.yarn.api.records.Token invalidNMToken =
+ tempManager.createNMToken(validAppAttemptId, validNode, user);
+ sb = new StringBuilder("Given NMToken for application : ");
+ sb.append(validAppAttemptId.toString())
+ .append(" seems to have been generated illegally.");
+ Assert.assertTrue(sb.toString().contains(
+ testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, invalidNMToken, true)));
+
+ // valid NMToken but belonging to other node
+ invalidNMToken =
+ nmTokenSecretManagerRM.createNMToken(validAppAttemptId, invalidNode,
+ user);
+ sb = new StringBuilder("Given NMToken for application : ");
+ sb.append(validAppAttemptId)
+ .append(" is not valid for current node manager.expected : ")
+ .append(validNode.toString())
+ .append(" found : ").append(invalidNode.toString());
+ Assert.assertTrue(sb.toString().contains(
+ testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, invalidNMToken, true)));
+
+ // using appAttempt-2 token for launching container for appAttempt-1.
+ invalidNMToken =
+ nmTokenSecretManagerRM.createNMToken(invalidAppAttemptId, validNode,
+ user);
+ sb = new StringBuilder("\nNMToken for application attempt : ");
+ sb.append(invalidAppAttemptId.toString())
+ .append(" was used for starting container with container token")
+ .append(" issued for application attempt : ")
+ .append(validAppAttemptId.toString());
+ Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, invalidNMToken, true).contains(sb.toString()));
+
+ // using correct tokens. nmtoken for appattempt should get saved.
+ testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken,
+ validNMToken, false);
+ Assert.assertTrue(nmTokenSecretManagerNM
+ .isAppAttemptNMTokenKeyPresent(validAppAttemptId));
+
+ // Rolling over master key twice so that we can check whether older keys
+ // are used for authentication.
+ rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
+ // Key rolled over once.. rolling over again
+ rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
+
+ // trying get container status. Now saved nmToken should be used for
+ // authentication.
+ sb = new StringBuilder("Container ");
+ sb.append(validContainerId.toString());
+ sb.append(" is not handled by this NodeManager");
+ Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+ validContainerId, validNMToken, false).contains(sb.toString()));
+
}
- /**
- * This tests a malice user getting a proper token but then messing with it by
- * tampering with containerID/Resource etc.. His/her containers should be
- * rejected.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws YarnException
- */
- private void testMaliceUser() throws IOException, InterruptedException,
- YarnException {
-
- LOG.info("Running test for malice user");
-
- ResourceManager resourceManager = yarnCluster.getResourceManager();
-
- final YarnRPC yarnRPC = YarnRPC.create(conf);
-
- // Submit an application
- ApplicationId appID = resourceManager.getClientRMService()
- .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
- .getApplicationId();
- ApplicationMasterProtocol scheduler = submitAndRegisterApplication(resourceManager,
- yarnRPC, appID);
-
- // Now request a container.
- final Container allocatedContainer = requestAndGetContainer(scheduler,
- appID);
-
- // Now talk to the NM for launching the container with modified resource
-
- org.apache.hadoop.yarn.api.records.Token containerToken =
- allocatedContainer.getContainerToken();
- ContainerTokenIdentifier originalContainerTokenId =
- BuilderUtils.newContainerTokenIdentifier(containerToken);
-
- // Malice user modifies the resource amount
- Resource modifiedResource = BuilderUtils.newResource(2048, 1);
- ContainerTokenIdentifier modifiedIdentifier =
- new ContainerTokenIdentifier(originalContainerTokenId.getContainerID(),
- originalContainerTokenId.getNmHostAddress(), "testUser",
- modifiedResource, Long.MAX_VALUE,
- originalContainerTokenId.getMasterKeyId(),
- ResourceManager.clusterTimeStamp);
- Token modifiedToken =
- new Token(modifiedIdentifier.getBytes(),
- containerToken.getPassword().array(), new Text(
- containerToken.getKind()), new Text(containerToken.getService()));
- makeTamperedStartContainerCall(yarnRPC, allocatedContainer,
- modifiedIdentifier, modifiedToken);
-
- // Malice user modifies the container-Id
- ContainerId newContainerId =
- BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(originalContainerTokenId
- .getContainerID().getApplicationAttemptId().getApplicationId(), 1),
- originalContainerTokenId.getContainerID().getId() + 42);
- modifiedIdentifier =
- new ContainerTokenIdentifier(newContainerId,
- originalContainerTokenId.getNmHostAddress(), "testUser",
- originalContainerTokenId.getResource(), Long.MAX_VALUE,
- originalContainerTokenId.getMasterKeyId(),
- ResourceManager.clusterTimeStamp);
- modifiedToken =
- new Token(modifiedIdentifier.getBytes(),
- containerToken.getPassword().array(), new Text(
- containerToken.getKind()), new Text(containerToken.getService()));
- makeTamperedStartContainerCall(yarnRPC, allocatedContainer,
- modifiedIdentifier, modifiedToken);
-
- // Similarly messing with anything else will fail.
-
- KillApplicationRequest request = Records
- .newRecord(KillApplicationRequest.class);
- request.setApplicationId(appID);
- resourceManager.getClientRMService().forceKillApplication(request);
+ protected void waitForNMToReceiveNMTokenKey(
+ NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
+ throws InterruptedException {
+ int attempt = 60;
+ ContainerManagerImpl cm =
+ ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
+ while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
+ .getNodeId() == null) && attempt-- > 0) {
+ Thread.sleep(2000);
+ }
}
- private void makeTamperedStartContainerCall(final YarnRPC yarnRPC,
- final Container allocatedContainer,
- final ContainerTokenIdentifier modifiedIdentifier,
- Token modifiedToken) {
- final ContainerId containerID = allocatedContainer.getId();
- UserGroupInformation maliceUser = UserGroupInformation
- .createRemoteUser(containerID.toString());
- maliceUser.addToken(modifiedToken);
- maliceUser.doAs(new PrivilegedAction() {
- @Override
- public Void run() {
- ContainerManagementProtocol client = (ContainerManagementProtocol) yarnRPC.getProxy(
- ContainerManagementProtocol.class, NetUtils
- .createSocketAddr(allocatedContainer.getNodeId().toString()),
- conf);
+ protected void rollNMTokenMasterKey(
+ NMTokenSecretManagerInRM nmTokenSecretManagerRM,
+ NMTokenSecretManagerInNM nmTokenSecretManagerNM) throws Exception {
+ int oldKeyId = nmTokenSecretManagerRM.getCurrentKey().getKeyId();
+ nmTokenSecretManagerRM.rollMasterKey();
+ int interval = 40;
+ while (nmTokenSecretManagerNM.getCurrentKey().getKeyId() == oldKeyId
+ && interval-- > 0) {
+ Thread.sleep(1000);
+ }
+ nmTokenSecretManagerRM.activateNextMasterKey();
+ Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
+ == nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
+ }
- LOG.info("Going to contact NM: ilLegal request");
- StartContainerRequest request =
- Records.newRecord(StartContainerRequest.class);
- try {
- request.setContainerToken(allocatedContainer.getContainerToken());
- ContainerLaunchContext context =
- createContainerLaunchContextForTest(modifiedIdentifier);
- request.setContainerLaunchContext(context);
- client.startContainer(request);
- fail("Connection initiation with illegally modified "
- + "tokens is expected to fail.");
- } catch (YarnException e) {
- LOG.error("Got exception", e);
- fail("Cannot get a YARN remote exception as "
- + "it will indicate RPC success");
- } catch (Exception e) {
- Assert.assertEquals(
- javax.security.sasl.SaslException.class
- .getCanonicalName(), e.getClass().getCanonicalName());
- Assert.assertTrue(e
- .getMessage()
- .contains(
- "DIGEST-MD5: digest response format violation. "
- + "Mismatched response."));
- }
- return null;
+ private String testGetContainer(YarnRPC rpc,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ ContainerId containerId,
+ org.apache.hadoop.yarn.api.records.Token nmToken,
+ boolean isExceptionExpected) {
+ try {
+ getContainerStatus(rpc, nmToken, containerId, appAttemptId, nodeId,
+ isExceptionExpected);
+ if (isExceptionExpected) {
+ fail("Exception was expected!!");
}
- });
+ return "";
+ } catch (Exception e) {
+ e.printStackTrace();
+ return e.getMessage();
+ }
}
- private void testExpiredTokens() throws IOException, InterruptedException,
- YarnException {
-
- LOG.info("\n\nRunning test for malice user");
-
- ResourceManager resourceManager = yarnCluster.getResourceManager();
-
- final YarnRPC yarnRPC = YarnRPC.create(conf);
-
- // Submit an application
- final ApplicationId appID = resourceManager.getClientRMService()
- .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
- .getApplicationId();
- ApplicationMasterProtocol scheduler = submitAndRegisterApplication(resourceManager,
- yarnRPC, appID);
-
- // Now request a container.
- final Container allocatedContainer = requestAndGetContainer(scheduler,
- appID);
-
- // Now talk to the NM for launching the container with modified containerID
- final ContainerId containerID = allocatedContainer.getId();
-
- org.apache.hadoop.yarn.api.records.Token containerToken =
- allocatedContainer.getContainerToken();
- final ContainerTokenIdentifier tokenId =
- BuilderUtils.newContainerTokenIdentifier(containerToken);
-
- /////////// Test calls with expired tokens
- UserGroupInformation unauthorizedUser = UserGroupInformation
- .createRemoteUser(containerID.toString());
-
- RMContainerTokenSecretManager containerTokenSecreteManager =
- resourceManager.getRMContainerTokenSecretManager();
- final ContainerTokenIdentifier newTokenId =
- new ContainerTokenIdentifier(tokenId.getContainerID(),
- tokenId.getNmHostAddress(), tokenId.getApplicationSubmitter(),
- tokenId.getResource(), System.currentTimeMillis() - 1,
- containerTokenSecreteManager.getCurrentKey().getKeyId(),
- ResourceManager.clusterTimeStamp);
- final byte[] passowrd =
- containerTokenSecreteManager.createPassword(
- newTokenId);
- // Create a valid token by using the key from the RM.
- Token token =
- new Token(newTokenId.getBytes(), passowrd,
- new Text(containerToken.getKind()), new Text(
- containerToken.getService()));
-
- unauthorizedUser.addToken(token);
- unauthorizedUser.doAs(new PrivilegedAction() {
- @Override
- public Void run() {
- ContainerManagementProtocol client = (ContainerManagementProtocol) yarnRPC.getProxy(
- ContainerManagementProtocol.class, NetUtils
- .createSocketAddr(allocatedContainer.getNodeId().toString()),
- conf);
-
- LOG.info("Going to contact NM with expired token");
- ContainerLaunchContext context = createContainerLaunchContextForTest(newTokenId);
- StartContainerRequest request =
- Records.newRecord(StartContainerRequest.class);
- request.setContainerLaunchContext(context);
- allocatedContainer.setContainerToken(BuilderUtils.newContainerToken(
- allocatedContainer.getNodeId(), passowrd, newTokenId));
- request.setContainerToken(allocatedContainer.getContainerToken());
-
- //Calling startContainer with an expired token.
- try {
- client.startContainer(request);
- fail("Connection initiation with expired "
- + "token is expected to fail.");
- } catch (Throwable t) {
- LOG.info("Got exception : ", t);
- Assert.assertTrue(t.getMessage().contains(
- "This token is expired. current time is"));
- }
-
- // Try stopping a container - should not get an expiry error.
- StopContainerRequest stopRequest = Records.newRecord(StopContainerRequest.class);
- stopRequest.setContainerId(newTokenId.getContainerID());
- try {
- client.stopContainer(stopRequest);
- } catch (Throwable t) {
- fail("Stop Container call should have succeeded");
- }
-
- return null;
+ protected String testStartContainer(YarnRPC rpc,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ org.apache.hadoop.yarn.api.records.Token containerToken,
+ org.apache.hadoop.yarn.api.records.Token nmToken,
+ boolean isExceptionExpected) {
+ try {
+ startContainer(rpc, nmToken, containerToken, nodeId,
+ appAttemptId.toString());
+ if (isExceptionExpected){
+ fail("Exception was expected!!");
}
- });
- /////////// End of testing calls with expired tokens
-
- KillApplicationRequest request = Records
- .newRecord(KillApplicationRequest.class);
- request.setApplicationId(appID);
- resourceManager.getClientRMService().forceKillApplication(request);
+ return "";
+ } catch (Exception e) {
+ e.printStackTrace();
+ return e.getMessage();
+ }
}
- private ApplicationMasterProtocol submitAndRegisterApplication(
- ResourceManager resourceManager, final YarnRPC yarnRPC,
- ApplicationId appID) throws IOException,
- UnsupportedFileSystemException, YarnException,
- InterruptedException {
-
- // Use ping to simulate sleep on Windows.
- List cmd = Shell.WINDOWS ?
- Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") :
- Arrays.asList("sleep", "100");
-
- ContainerLaunchContext amContainer =
- BuilderUtils.newContainerLaunchContext(
- Collections. emptyMap(),
- new HashMap(), cmd,
- new HashMap(), null,
- new HashMap());
-
- ApplicationSubmissionContext appSubmissionContext = recordFactory
- .newRecordInstance(ApplicationSubmissionContext.class);
- appSubmissionContext.setApplicationId(appID);
- appSubmissionContext.setAMContainerSpec(amContainer);
- appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1));
-
- SubmitApplicationRequest submitRequest = recordFactory
- .newRecordInstance(SubmitApplicationRequest.class);
- submitRequest.setApplicationSubmissionContext(appSubmissionContext);
- resourceManager.getClientRMService().submitApplication(submitRequest);
+ private void
+ getContainerStatus(YarnRPC rpc,
+ org.apache.hadoop.yarn.api.records.Token nmToken,
+ ContainerId containerId,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ boolean isExceptionExpected) throws Exception {
+ GetContainerStatusRequest request =
+ Records.newRecord(GetContainerStatusRequest.class);
+ request.setContainerId(containerId);
+
+ ContainerManagementProtocol proxy = null;
+
+ try {
+ proxy =
+ getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
+ appAttemptId.toString());
+ proxy.getContainerStatus(request);
+
+ } finally {
+ if (proxy != null) {
+ rpc.stopProxy(proxy, conf);
+ }
+ }
+ }
+
+ private void startContainer(final YarnRPC rpc,
+ org.apache.hadoop.yarn.api.records.Token nmToken,
+ org.apache.hadoop.yarn.api.records.Token containerToken,
+ NodeId nodeId, String user) throws Exception {
+
+ StartContainerRequest request =
+ Records.newRecord(StartContainerRequest.class);
+ request.setContainerToken(containerToken);
+ ContainerLaunchContext context =
+ Records.newRecord(ContainerLaunchContext.class);
+ request.setContainerLaunchContext(context);
- // Wait till container gets allocated for AM
- int waitCounter = 0;
- RMApp app = resourceManager.getRMContext().getRMApps().get(appID);
- RMAppAttempt appAttempt = app == null ? null : app.getCurrentAppAttempt();
- RMAppAttemptState state = appAttempt == null ? null : appAttempt
- .getAppAttemptState();
- while ((app == null || appAttempt == null || state == null || !state
- .equals(RMAppAttemptState.LAUNCHED))
- && waitCounter++ != 20) {
- LOG.info("Waiting for applicationAttempt to be created.. ");
- Thread.sleep(1000);
- app = resourceManager.getRMContext().getRMApps().get(appID);
- appAttempt = app == null ? null : app.getCurrentAppAttempt();
- state = appAttempt == null ? null : appAttempt.getAppAttemptState();
+ ContainerManagementProtocol proxy = null;
+ try {
+ proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
+ proxy.startContainer(request);
+ } finally {
+ if (proxy != null) {
+ rpc.stopProxy(proxy, conf);
+ }
}
- Assert.assertNotNull(app);
- Assert.assertNotNull(appAttempt);
- Assert.assertNotNull(state);
- Assert.assertEquals(RMAppAttemptState.LAUNCHED, state);
+ }
- UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(
- appAttempt.getAppAttemptId().toString());
+ protected ContainerManagementProtocol getContainerManagementProtocolProxy(
+ final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken,
+ NodeId nodeId, String user) {
+ ContainerManagementProtocol proxy;
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ final InetSocketAddress addr =
+ NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
+ ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+
+ proxy = ugi
+ .doAs(new PrivilegedAction() {
- // Ask for a container from the RM
- final InetSocketAddress schedulerAddr =
- resourceManager.getApplicationMasterService().getBindAddress();
- if (UserGroupInformation.isSecurityEnabled()) {
- AMRMTokenIdentifier appTokenIdentifier = new AMRMTokenIdentifier(
- appAttempt.getAppAttemptId());
- AMRMTokenSecretManager appTokenSecretManager =
- new AMRMTokenSecretManager(conf);
- appTokenSecretManager.setMasterKey(resourceManager
- .getAMRMTokenSecretManager().getMasterKey());
- Token appToken =
- new Token(appTokenIdentifier,
- appTokenSecretManager);
- SecurityUtil.setTokenService(appToken, schedulerAddr);
- currentUser.addToken(appToken);
- }
-
- ApplicationMasterProtocol scheduler = currentUser
- .doAs(new PrivilegedAction() {
@Override
- public ApplicationMasterProtocol run() {
- return (ApplicationMasterProtocol) yarnRPC.getProxy(ApplicationMasterProtocol.class,
- schedulerAddr, conf);
+ public ContainerManagementProtocol run() {
+ return (ContainerManagementProtocol) rpc.getProxy(
+ ContainerManagementProtocol.class,
+ addr, conf);
}
});
-
- // Register the appMaster
- RegisterApplicationMasterRequest request = recordFactory
- .newRecordInstance(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(resourceManager.getRMContext()
- .getRMApps().get(appID).getCurrentAppAttempt().getAppAttemptId());
- scheduler.registerApplicationMaster(request);
- return scheduler;
+ return proxy;
}
- private Container requestAndGetContainer(ApplicationMasterProtocol scheduler,
- ApplicationId appID) throws YarnException, InterruptedException,
- IOException {
-
- // Request a container allocation.
- List ask = new ArrayList();
- ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
- ResourceRequest.ANY, BuilderUtils.newResource(1024, 1), 1));
-
- AllocateRequest allocateRequest = AllocateRequest.newInstance(
- BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask,
- new ArrayList(), null);
- List allocatedContainers = scheduler.allocate(allocateRequest)
- .getAllocatedContainers();
-
- // Modify ask to request no more.
- allocateRequest.setAskList(new ArrayList());
-
- int waitCounter = 0;
- while ((allocatedContainers == null || allocatedContainers.size() == 0)
- && waitCounter++ != 20) {
- LOG.info("Waiting for container to be allocated..");
- Thread.sleep(1000);
- allocateRequest.setResponseId(allocateRequest.getResponseId() + 1);
- allocatedContainers = scheduler.allocate(allocateRequest)
- .getAllocatedContainers();
- }
-
- Assert.assertNotNull("Container is not allocted!", allocatedContainers);
- Assert.assertEquals("Didn't get one container!", 1, allocatedContainers
- .size());
+ /**
+ * This tests a malice user getting a proper token but then messing with it by
+ * tampering with containerID/Resource etc.. His/her containers should be
+ * rejected.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws YarnException
+ */
+ private void testContainerToken(Configuration conf) throws IOException,
+ InterruptedException, YarnException {
- return allocatedContainers.get(0);
- }
+ LOG.info("Running test for malice user");
+ /*
+ * We need to check for containerToken (authorization).
+ * Here we will be assuming that we have valid NMToken
+ * 1) ContainerToken used is expired.
+ * 2) ContainerToken is tampered (resource is modified).
+ */
+ NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
+ yarnCluster.getResourceManager().getRMContext()
+ .getNMTokenSecretManager();
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
+ NodeManager nm = yarnCluster.getNodeManager(0);
+ NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
+ nm.getNMContext().getNMTokenSecretManager();
+ String user = "test";
+
+ waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
- private ContainerLaunchContext createContainerLaunchContextForTest(
- ContainerTokenIdentifier tokenId) {
- ContainerLaunchContext context =
- BuilderUtils.newContainerLaunchContext(
- new HashMap(),
- new HashMap(), new ArrayList(),
- new HashMap(), null,
- new HashMap());
- return context;
+ NodeId nodeId = nm.getNMContext().getNodeId();
+
+ // Both id should be equal.
+ Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
+ nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
+
+ // Creating a tampered Container Token
+ RMContainerTokenSecretManager containerTokenSecretManager =
+ yarnCluster.getResourceManager().getRMContainerTokenSecretManager();
+
+ RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
+ new RMContainerTokenSecretManager(conf);
+ tamperedContainerTokenSecretManager.rollMasterKey();
+ do {
+ tamperedContainerTokenSecretManager.rollMasterKey();
+ tamperedContainerTokenSecretManager.activateNextMasterKey();
+ } while (containerTokenSecretManager.getCurrentKey().getKeyId()
+ == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
+
+ Resource r = Resource.newInstance(1230, 2);
+ // Creating modified containerToken
+ Token containerToken =
+ tamperedContainerTokenSecretManager.createContainerToken(cId, nodeId,
+ user, r);
+ Token nmToken =
+ nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
+ YarnRPC rpc = YarnRPC.create(conf);
+ StringBuilder sb = new StringBuilder("Given Container ");
+ sb.append(cId);
+ sb.append(" seems to have an illegally generated token.");
+ Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
+ containerToken, nmToken, true).contains(sb.toString()));
}
}