diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java index d13159b..cef5a62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java @@ -35,14 +35,16 @@ private final String user; private final ContainerId containerId; private final Resource resource; + private final boolean isAMContainer; @Private @Unstable public ContainerContext(String user, ContainerId containerId, - Resource resource) { + Resource resource, boolean isAMContainer) { this.user = user; this.containerId = containerId; this.resource = resource; + this.isAMContainer = isAMContainer; } /** @@ -72,4 +74,13 @@ public ContainerId getContainerId() { public Resource getResource() { return resource; } + + /** + * Get whether the container is an AM or not. + * + * @return true if the container is an AM, false otherwise. + */ + public boolean isAMContainer() { + return isAMContainer; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java index 5b5bbda..ca129b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java @@ -37,8 +37,8 @@ @Private @Unstable public ContainerInitializationContext(String user, ContainerId containerId, - Resource resource) { - super(user, containerId, resource); + Resource resource, boolean isAMContainer) { + super(user, containerId, resource, isAMContainer); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java index 34ba73e..4a56f64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java @@ -37,8 +37,8 @@ @Private @Unstable public ContainerTerminationContext(String user, ContainerId containerId, - Resource resource) { - super(user, containerId, resource); + Resource resource, boolean isAMContainer) { + super(user, containerId, resource, isAMContainer); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java index 9a60d01..137c1db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java @@ -66,13 +66,14 @@ public ContainerTokenIdentifier(ContainerId containerID, int masterKeyId, long rmIdentifier, Priority priority, long creationTime) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, - CommonNodeLabelsManager.NO_LABEL); + CommonNodeLabelsManager.NO_LABEL, false); } public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, - LogAggregationContext logAggregationContext, String nodeLabelExpression) { + LogAggregationContext logAggregationContext, String nodeLabelExpression, + boolean isAMContainer) { ContainerTokenIdentifierProto.Builder builder = ContainerTokenIdentifierProto.newBuilder(); if (containerID != null) { @@ -99,6 +100,7 @@ public ContainerTokenIdentifier(ContainerId containerID, String hostName, if (nodeLabelExpression != null) { builder.setNodeLabelExpression(nodeLabelExpression); } + builder.setIsAMContainer(isAMContainer); proto = builder.build(); } @@ -168,6 +170,15 @@ public LogAggregationContext getLogAggregationContext() { return new LogAggregationContextPBImpl(proto.getLogAggregationContext()); } + /** + * Get whether the container is an AM or not. + * + * @return true if the container is an AM, false otherwise. + */ + public boolean getIsAMContainer() { + return proto.getIsAMContainer(); + } + @Override public void write(DataOutput out) throws IOException { LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto index d1bef21..ac8604c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto @@ -50,6 +50,7 @@ message ContainerTokenIdentifierProto { optional int64 creationTime = 9; optional LogAggregationContextProto logAggregationContext = 10; optional string nodeLabelExpression = 11; + optional bool isAMContainer = 12 [default = false]; } message ClientToAMTokenIdentifierProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index fb6f79b..78ed1a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -225,7 +225,8 @@ public void handle(AuxServicesEvent event) { try { serv.initializeContainer(new ContainerInitializationContext( event.getUser(), event.getContainer().getContainerId(), - event.getContainer().getResource())); + event.getContainer().getResource(), event.getContainer() + .getContainerTokenIdentifier().getIsAMContainer())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_INIT, th); @@ -237,7 +238,8 @@ public void handle(AuxServicesEvent event) { try { serv.stopContainer(new ContainerTerminationContext( event.getUser(), event.getContainer().getContainerId(), - event.getContainer().getResource())); + event.getContainer().getResource(), event.getContainer() + .getContainerTokenIdentifier().getIsAMContainer())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_STOP, th); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 7bdfdfb..aed78ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -896,7 +896,7 @@ public static Token createContainerToken(ContainerId cId, long rmIdentifier, ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), user, r, System.currentTimeMillis() + 100000L, 123, rmIdentifier, - Priority.newInstance(0), 0, logAggregationContext, null); + Priority.newInstance(0), 0, logAggregationContext, null, false); Token containerToken = BuilderUtils .newContainerToken(nodeId, containerTokenSecretManager diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index dbc3cb5..58fd880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -467,13 +467,21 @@ public ContainersAndNMTokensAllocation(List containerList, .hasNext();) { RMContainer rmContainer = i.next(); Container container = rmContainer.getContainer(); + boolean isAMContainer = false; + // The working knowledge is that masterContainer for AM is null as it + // itself is the master container. + if (rmContext.getRMApps() + .get(container.getId().getApplicationAttemptId().getApplicationId()) + .getCurrentAppAttempt().getMasterContainer() == null) { + isAMContainer = true; + } try { // create container token and NMToken altogether. container.setContainerToken(rmContext.getContainerTokenSecretManager() .createContainerToken(container.getId(), container.getNodeId(), getUser(), container.getResource(), container.getPriority(), rmContainer.getCreationTime(), this.logAggregationContext, - rmContainer.getNodeLabelExpression())); + rmContainer.getNodeLabelExpression(), isAMContainer)); NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index 1c0533d..b923aa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -179,7 +179,7 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime) { return createContainerToken(containerId, nodeId, appSubmitter, capability, - priority, createTime, null, null); + priority, createTime, null, null, false); } /** @@ -192,12 +192,13 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, * @param priority * @param createTime * @param logAggregationContext + * @param isAMContainer * @return the container-token */ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, - String nodeLabelExpression) { + String nodeLabelExpression, boolean isAMContainer) { byte[] password; ContainerTokenIdentifier tokenIdentifier; long expiryTimeStamp = @@ -211,7 +212,7 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, appSubmitter, capability, expiryTimeStamp, this.currentMasterKey .getMasterKey().getKeyId(), ResourceManager.getClusterTimeStamp(), priority, createTime, - logAggregationContext, nodeLabelExpression); + logAggregationContext, nodeLabelExpression, isAMContainer); password = this.createPassword(tokenIdentifier); } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 0ea993f..ed30fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -286,11 +286,12 @@ protected RMContainerTokenSecretManager createContainerTokenSecretManager( public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, - LogAggregationContext logAggregationContext, String nodeLabelExp) { + LogAggregationContext logAggregationContext, String nodeLabelExp, + boolean isAMContainer) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext, - nodeLabelExp); + nodeLabelExp, isAMContainer); } }; }