diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java index 2c2238f..9d3abb2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java @@ -31,6 +31,7 @@ * It provides details such as: *
ExecutionType of the container.
+ * @return ExecutionType of the container
+ */
+ @Public
+ @Stable
+ public abstract ExecutionType getExecutionType();
+
+ @Private
+ @Unstable
+ public abstract void setExecutionType(ExecutionType executionType);
+
+ /**
* Get the ContainerState of the container.
* @return ContainerState of the container
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionType.java
new file mode 100644
index 0000000..75346f3
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ExecutionType.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
+/**
+ * Container and resource request property encoding execution semantics.
+ *
+ * + * The execution types are the following: + *
ExecutionType of the requested container.
+ *
+ * @param execType
+ * ExecutionType of the requested container
+ */
+ @Public
+ @Stable
+ public abstract void setExecutionType(ExecutionType execType);
+
+ /**
+ * Get whether locality relaxation is enabled with this
+ * ResourceRequest. Defaults to true.
+ *
+ * @return whether locality relaxation is enabled with this
+ * ResourceRequest.
+ */
+ @Public
+ @Stable
+ public abstract ExecutionType getExecutionType();
+
/**
* For a request at a network hierarchy level, set whether locality can be relaxed * to that level and beyond.
@@ -322,6 +353,14 @@ public boolean equals(Object obj) { return false; } else if (!priority.equals(other.getPriority())) return false; + ExecutionType executionType = getExecutionType(); + if (executionType == null) { + if (other.getExecutionType() != null) { + return false; + } + } else if (executionType != other.getExecutionType()) { + return false; + } if (getNodeLabelExpression() == null) { if (other.getNodeLabelExpression() != null) { return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java index f7a9b02..d7ccd8b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; /** @@ -36,22 +37,26 @@ private final ContainerId containerId; private final Resource resource; private final ContainerType containerType; + private final ExecutionType executionType; @Private @Unstable public ContainerContext(String user, ContainerId containerId, Resource resource) { - this(user, containerId, resource, ContainerType.TASK); + this(user, containerId, resource, ContainerType.TASK, + ExecutionType.GUARANTEED); } @Private @Unstable public ContainerContext(String user, ContainerId containerId, - Resource resource, ContainerType containerType) { + Resource resource, ContainerType containerType, + ExecutionType executionType) { this.user = user; this.containerId = containerId; this.resource = resource; this.containerType = containerType; + this.executionType = executionType; } /** @@ -91,4 +96,14 @@ public Resource getResource() { public ContainerType getContainerType() { return containerType; } + + /** + * Get {@link ExecutionType} the execution type of the container + * being initialized or stopped. + * + * @return the execution type of the container + */ + public ExecutionType getExecutionType() { + return executionType; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java index 44428f9..1e8db17 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; /** @@ -44,8 +45,9 @@ public ContainerInitializationContext(String user, ContainerId containerId, @Private @Unstable public ContainerInitializationContext(String user, ContainerId containerId, - Resource resource, ContainerType containerType) { - super(user, containerId, resource, containerType); + Resource resource, ContainerType containerType, + ExecutionType executionType) { + super(user, containerId, resource, containerType, executionType); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java index 2e4ad3b..e00466e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; /** @@ -44,8 +45,9 @@ public ContainerTerminationContext(String user, ContainerId containerId, @Private @Unstable public ContainerTerminationContext(String user, ContainerId containerId, - Resource resource, ContainerType containerType) { - super(user, containerId, resource, containerType); + Resource resource, ContainerType containerType, + ExecutionType executionType) { + super(user, containerId, resource, containerType, executionType); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 09d2bd5..f6b7466 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -275,6 +275,11 @@ enum ContainerTypeProto { TASK = 2; } +enum ExecutionTypeProto { + GUARANTEED = 1; + QUEUEABLE = 2; +} + //////////////////////////////////////////////////////////////////////// ////// From AM_RM_Protocol ///////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// @@ -285,6 +290,7 @@ message ResourceRequestProto { optional int32 num_containers = 4; optional bool relax_locality = 5 [default = true]; optional string node_label_expression = 6; + optional ExecutionTypeProto executionType = 7 [default = GUARANTEED]; } enum AMCommandProto { @@ -480,6 +486,7 @@ message ContainerStatusProto { optional string diagnostics = 3 [default = "N/A"]; optional int32 exit_status = 4 [default = -1000]; optional ResourceProto capability = 5; + optional ExecutionTypeProto executionType = 6 [default = GUARANTEED]; } enum ContainerExitStatusProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index e0bf2d3..73c2725 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -108,6 +109,7 @@ protected AMRMClient(String name) { final Priority priority; final boolean relaxLocality; final String nodeLabelsExpression; + final ExecutionType executionType; /** * Instantiates a {@link ContainerRequest} with the given constraints and @@ -152,6 +154,33 @@ public ContainerRequest(Resource capability, String[] nodes, String[] racks, Priority priority, boolean relaxLocality) { this(capability, nodes, racks, priority, relaxLocality, null); } + + /** + * Instantiates a {@link ContainerRequest} with the given constraints. + * + * @param capability + * The {@link Resource} to be requested for each container. + * @param nodes + * Any hosts to request that the containers are placed on. + * @param racks + * Any racks to request that the containers are placed on. The + * racks corresponding to any hosts requested will be automatically + * added to this list. + * @param priority + * The priority at which to request the containers. Higher + * priorities have lower numerical values. + * @param relaxLocality + * If true, containers for this request may be assigned on hosts + * and racks other than the ones explicitly requested. + * @param nodeLabelsExpression + * Set node labels to allocate resource, now we only support + * asking for only a single node label + */ + public ContainerRequest(Resource capability, String[] nodes, String[] racks, + Priority priority, boolean relaxLocality, String nodeLabelsExpression) { + this(capability, nodes, racks, priority, relaxLocality, null, + ExecutionType.GUARANTEED); + } /** * Instantiates a {@link ContainerRequest} with the given constraints. @@ -173,10 +202,12 @@ public ContainerRequest(Resource capability, String[] nodes, * @param nodeLabelsExpression * Set node labels to allocate resource, now we only support * asking for only a single node label + * @param executionType + * Set the execution type of the container request. */ - public ContainerRequest(Resource capability, String[] nodes, - String[] racks, Priority priority, boolean relaxLocality, - String nodeLabelsExpression) { + public ContainerRequest(Resource capability, String[] nodes, String[] racks, + Priority priority, boolean relaxLocality, String nodeLabelsExpression, + ExecutionType executionType) { // Validate request Preconditions.checkArgument(capability != null, "The Resource to be requested for each container " + @@ -194,6 +225,7 @@ public ContainerRequest(Resource capability, String[] nodes, this.priority = priority; this.relaxLocality = relaxLocality; this.nodeLabelsExpression = nodeLabelsExpression; + this.executionType = executionType; } public Resource getCapability() { @@ -220,10 +252,15 @@ public String getNodeLabelExpression() { return nodeLabelsExpression; } + public ExecutionType getExecutionType() { + return executionType; + } + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Capability[").append(capability).append("]"); sb.append("Priority[").append(priority).append("]"); + sb.append("ExecutionType[").append(executionType).append("]"); return sb.toString(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java index d33d06d..1eac677 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java @@ -24,15 +24,18 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder; import com.google.protobuf.TextFormat; + @Private @Unstable public class ContainerStatusPBImpl extends ContainerStatus { @@ -79,6 +82,7 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("ContainerStatus: ["); sb.append("ContainerId: ").append(getContainerId()).append(", "); + sb.append("ExecutionType: ").append(getExecutionType()).append(", "); sb.append("State: ").append(getState()).append(", "); sb.append("Capability: ").append(getCapability()).append(", "); sb.append("Diagnostics: ").append(getDiagnostics()).append(", "); @@ -107,7 +111,25 @@ private synchronized void maybeInitBuilder() { } viaProto = false; } - + + @Override + public synchronized ExecutionType getExecutionType() { + ContainerStatusProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasExecutionType()) { + return null; + } + return convertFromProtoFormat(p.getExecutionType()); + } + + @Override + public synchronized void setExecutionType(ExecutionType executionType) { + maybeInitBuilder(); + if (executionType == null) { + builder.clearExecutionType(); + return; + } + builder.setExecutionType(convertToProtoFormat(executionType)); + } @Override public synchronized ContainerState getState() { @@ -206,6 +228,14 @@ private ContainerIdProto convertToProtoFormat(ContainerId t) { return ((ContainerIdPBImpl)t).getProto(); } + private ExecutionType convertFromProtoFormat(ExecutionTypeProto e) { + return ProtoUtils.convertFromProtoFormat(e); + } + + private ExecutionTypeProto convertToProtoFormat(ExecutionType e) { + return ProtoUtils.convertToProtoFormat(e); + } + private ResourceProto convertToProtoFormat(Resource e) { return ((ResourcePBImpl)e).getProto(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index e742f4c..29ed0f3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -54,6 +55,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.server.api.ContainerType; @@ -282,4 +284,14 @@ public static ContainerTypeProto convertToProtoFormat(ContainerType e) { public static ContainerType convertFromProtoFormat(ContainerTypeProto e) { return ContainerType.valueOf(e.name()); } + + /* + * ExecutionType + */ + public static ExecutionTypeProto convertToProtoFormat(ExecutionType e) { + return ExecutionTypeProto.valueOf(e.name()); + } + public static ExecutionType convertFromProtoFormat(ExecutionTypeProto e) { + return ExecutionType.valueOf(e.name()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index fc09cef..53ae2cd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -206,4 +207,24 @@ public void setNodeLabelExpression(String nodeLabelExpression) { } builder.setNodeLabelExpression(nodeLabelExpression); } + + @Override + public ExecutionType getExecutionType() { + ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasExecutionType()) { + return null; + } + return ProtoUtils.convertFromProtoFormat(p.getExecutionType()); + } + + @Override + public void setExecutionType(ExecutionType execType) { + maybeInitBuilder(); + if (execType == null) { + builder.clearExecutionType(); + return; + } + builder.setExecutionType(ProtoUtils.convertToProtoFormat(execType)); + } + } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java index 106e6d5..9c69d29 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto; import org.apache.hadoop.yarn.server.api.ContainerType; @@ -68,7 +70,8 @@ public ContainerTokenIdentifier(ContainerId containerID, int masterKeyId, long rmIdentifier, Priority priority, long creationTime) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, - CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK); + CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK, + ExecutionType.GUARANTEED); } public ContainerTokenIdentifier(ContainerId containerID, String hostName, @@ -77,14 +80,14 @@ public ContainerTokenIdentifier(ContainerId containerID, String hostName, LogAggregationContext logAggregationContext, String nodeLabelExpression) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, logAggregationContext, - nodeLabelExpression, ContainerType.TASK); + nodeLabelExpression, ContainerType.TASK, ExecutionType.GUARANTEED); } public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext, String nodeLabelExpression, - ContainerType containerType) { + ContainerType containerType, ExecutionType executionType) { ContainerTokenIdentifierProto.Builder builder = ContainerTokenIdentifierProto.newBuilder(); if (containerID != null) { @@ -112,6 +115,7 @@ public ContainerTokenIdentifier(ContainerId containerID, String hostName, builder.setNodeLabelExpression(nodeLabelExpression); } builder.setContainerType(convertToProtoFormat(containerType)); + builder.setExecutionType(convertToProtoFormat(executionType)); proto = builder.build(); } @@ -181,6 +185,17 @@ public ContainerType getContainerType(){ return convertFromProtoFormat(proto.getContainerType()); } + /** + * Get the ExecutionType of container to allocate + * @return ExecutionType + */ + public ExecutionType getExecutionType(){ + if (!proto.hasExecutionType()) { + return null; + } + return convertFromProtoFormat(proto.getExecutionType()); + } + public ContainerTokenIdentifierProto getProto() { return proto; } @@ -265,4 +280,13 @@ private ContainerType convertFromProtoFormat( ContainerTypeProto containerType) { return ProtoUtils.convertFromProtoFormat(containerType); } + + private ExecutionTypeProto convertToProtoFormat(ExecutionType executionType) { + return ProtoUtils.convertToProtoFormat(executionType); + } + + private ExecutionType convertFromProtoFormat( + ExecutionTypeProto executionType) { + return ProtoUtils.convertFromProtoFormat(executionType); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto index 339e99e..2403fc1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto @@ -51,6 +51,7 @@ message ContainerTokenIdentifierProto { optional LogAggregationContextProto logAggregationContext = 10; optional string nodeLabelExpression = 11; optional ContainerTypeProto containerType = 12; + optional ExecutionTypeProto executionType = 13; } message ClientToAMTokenIdentifierProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java index 68f0b9d..cb9a683 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -209,6 +210,9 @@ public void testContainerTokenIdentifier() throws IOException { Assert.assertEquals(ContainerType.TASK, anotherToken.getContainerType()); + + Assert.assertEquals(ExecutionType.GUARANTEED, + anotherToken.getExecutionType()); } @Test @@ -369,10 +373,10 @@ public void testAMContainerTokenIdentifier() throws IOException { Priority priority = Priority.newInstance(1); long creationTime = 1000; - ContainerTokenIdentifier token = - new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, - expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, - null, CommonNodeLabelsManager.NO_LABEL, ContainerType.APPLICATION_MASTER); + ContainerTokenIdentifier token = new ContainerTokenIdentifier(containerID, + hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, + priority, creationTime, null, CommonNodeLabelsManager.NO_LABEL, + ContainerType.APPLICATION_MASTER, ExecutionType.GUARANTEED); ContainerTokenIdentifier anotherToken = new ContainerTokenIdentifier(); @@ -384,10 +388,13 @@ public void testAMContainerTokenIdentifier() throws IOException { Assert.assertEquals(ContainerType.APPLICATION_MASTER, anotherToken.getContainerType()); - token = - new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, - expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, - null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK); + Assert.assertEquals(ExecutionType.GUARANTEED, + anotherToken.getExecutionType()); + + token = new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, + expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, + null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK, + ExecutionType.QUEUEABLE); anotherToken = new ContainerTokenIdentifier(); @@ -398,6 +405,9 @@ public void testAMContainerTokenIdentifier() throws IOException { Assert.assertEquals(ContainerType.TASK, anotherToken.getContainerType()); + + Assert.assertEquals(ExecutionType.QUEUEABLE, + anotherToken.getExecutionType()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index cd5ed88..dc9a4cd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -226,7 +226,9 @@ public void handle(AuxServicesEvent event) { serv.initializeContainer(new ContainerInitializationContext( event.getUser(), event.getContainer().getContainerId(), event.getContainer().getResource(), event.getContainer() - .getContainerTokenIdentifier().getContainerType())); + .getContainerTokenIdentifier().getContainerType(), + event.getContainer().getContainerTokenIdentifier() + .getExecutionType())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_INIT, th); @@ -239,7 +241,9 @@ public void handle(AuxServicesEvent event) { serv.stopContainer(new ContainerTerminationContext( event.getUser(), event.getContainer().getContainerId(), event.getContainer().getResource(), event.getContainer() - .getContainerTokenIdentifier().getContainerType())); + .getContainerTokenIdentifier().getContainerType(), + event.getContainer().getContainerTokenIdentifier() + .getExecutionType())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_STOP, th); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 101fef0..51d21a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -1839,7 +1840,8 @@ private ContainerId createContainer(ApplicationAttemptId appAttemptId1, ContainerTokenIdentifier containerToken = new ContainerTokenIdentifier( containerId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 100000L, 123, DUMMY_RM_IDENTIFIER, - Priority.newInstance(0), 0, null, null, containerType); + Priority.newInstance(0), 0, null, null, containerType, + ExecutionType.GUARANTEED); Container container = mock(Container.class); context.getContainers().put(containerId, container); when(container.getContainerTokenIdentifier()).thenReturn(containerToken); @@ -1878,6 +1880,7 @@ private ContainerId finishContainer(ApplicationId application1, return containerIds; } + @SuppressWarnings("unchecked") private void finishApplication(ApplicationId appId, LogAggregationService logAggregationService) throws Exception { dispatcher.await(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index c5f8def..20e6066 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; @@ -498,7 +499,8 @@ private Container updateContainerAndNMToken(RMContainer rmContainer, .createContainerToken(container.getId(), container.getNodeId(), getUser(), container.getResource(), container.getPriority(), rmContainer.getCreationTime(), this.logAggregationContext, - rmContainer.getNodeLabelExpression(), containerType)); + rmContainer.getNodeLabelExpression(), containerType, + ExecutionType.GUARANTEED)); NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index 6f00615..e859e70 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; @@ -180,7 +181,8 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime) { return createContainerToken(containerId, nodeId, appSubmitter, capability, - priority, createTime, null, null, ContainerType.TASK); + priority, createTime, null, null, ContainerType.TASK, + ExecutionType.GUARANTEED); } /** @@ -195,12 +197,14 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, * @param logAggregationContext * @param nodeLabelExpression * @param containerType + * @param executionType * @return the container-token */ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, - String nodeLabelExpression, ContainerType containerType) { + String nodeLabelExpression, ContainerType containerType, + ExecutionType executionType) { byte[] password; ContainerTokenIdentifier tokenIdentifier; long expiryTimeStamp = @@ -209,12 +213,12 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, // Lock so that we use the same MasterKey's keyId and its bytes this.readLock.lock(); try { - tokenIdentifier = - new ContainerTokenIdentifier(containerId, nodeId.toString(), - appSubmitter, capability, expiryTimeStamp, this.currentMasterKey - .getMasterKey().getKeyId(), - ResourceManager.getClusterTimeStamp(), priority, createTime, - logAggregationContext, nodeLabelExpression, containerType); + tokenIdentifier = new ContainerTokenIdentifier(containerId, + nodeId.toString(), appSubmitter, capability, expiryTimeStamp, + this.currentMasterKey.getMasterKey().getKeyId(), + ResourceManager.getClusterTimeStamp(), priority, createTime, + logAggregationContext, nodeLabelExpression, containerType, + executionType); password = this.createPassword(tokenIdentifier); } finally { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index b5b2222..5ffe14e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; @@ -290,11 +291,12 @@ protected RMContainerTokenSecretManager createContainerTokenSecretManager( public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, - LogAggregationContext logAggregationContext, String nodeLabelExp, ContainerType containerType) { + LogAggregationContext logAggregationContext, String nodeLabelExp, + ContainerType containerType, ExecutionType executionType) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext, - nodeLabelExp, containerType); + nodeLabelExp, containerType, executionType); } }; }