diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 38fa8b9..edf6e86 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -66,6 +66,15 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority, Token containerToken) { + return newInstance(containerId, nodeId, nodeHttpAddress, resource, priority, + containerToken, ExecutionType.GUARANTEED); + } + + @Private + @Unstable + public static Container newInstance(ContainerId containerId, NodeId nodeId, + String nodeHttpAddress, Resource resource, Priority priority, + Token containerToken, ExecutionType executionType) { Container container = Records.newRecord(Container.class); container.setId(containerId); container.setNodeId(nodeId); @@ -73,6 +82,7 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, container.setResource(resource); container.setPriority(priority); container.setContainerToken(containerToken); + container.setExecutionType(executionType); return container; } @@ -163,4 +173,19 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, @Private @Unstable public abstract void setContainerToken(Token containerToken); + + /** + * Get the ExecutionType for the container. + * @return ExecutionType for the container. + */ + @Private + @Unstable + public abstract ExecutionType getExecutionType(); + + /** + * Set the ExecutionType for the container. + */ + @Private + @Unstable + public abstract void setExecutionType(ExecutionType executionType); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 60cdfd1..159a174 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -92,6 +92,7 @@ message ContainerProto { optional ResourceProto resource = 4; optional PriorityProto priority = 5; optional hadoop.common.TokenProto container_token = 6; + optional ExecutionTypeProto execution_type = 7; } message ContainerReportProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index 1700068..6ef6c30 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; @Private @Unstable @@ -47,7 +49,8 @@ private Resource resource = null; private Priority priority = null; private Token containerToken = null; - + private ExecutionType executionType = null; + public ContainerPBImpl() { builder = ContainerProto.newBuilder(); } @@ -248,6 +251,27 @@ public void setContainerToken(Token containerToken) { this.containerToken = containerToken; } + @Override + public ExecutionType getExecutionType() { + ContainerProtoOrBuilder p = viaProto ? proto : builder; + if (this.executionType != null) { + return this.executionType; + } + if (!p.hasExecutionType()) { + return null; + } + this.executionType = convertFromProtoFormat(p.getExecutionType()); + return this.executionType; + } + + @Override + public void setExecutionType(ExecutionType executionType) { + maybeInitBuilder(); + if (executionType == null) + builder.clearExecutionType(); + this.executionType = executionType; + } + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { return new ContainerIdPBImpl(p); } @@ -288,6 +312,15 @@ private TokenProto convertToProtoFormat(Token t) { return ((TokenPBImpl)t).getProto(); } + private ExecutionTypeProto convertToProtoFormat(ExecutionType executionType) { + return ProtoUtils.convertToProtoFormat(executionType); + } + + private ExecutionType convertFromProtoFormat( + ExecutionTypeProto executionType) { + return ProtoUtils.convertFromProtoFormat(executionType); + } + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Container: ["); @@ -297,6 +330,7 @@ public String toString() { sb.append("Resource: ").append(getResource()).append(", "); sb.append("Priority: ").append(getPriority()).append(", "); sb.append("Token: ").append(getContainerToken()).append(", "); + sb.append("ExecutionType: ").append(getExecutionType()).append(", "); sb.append("]"); return sb.toString(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index a70d143..b2c455f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -236,7 +236,7 @@ public static ContainerStatus newContainerStatus(ContainerId containerId, public static Container newContainer(ContainerId containerId, NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority, - Token containerToken) { + Token containerToken, ExecutionType executionType) { Container container = recordFactory.newRecordInstance(Container.class); container.setId(containerId); container.setNodeId(nodeId); @@ -244,6 +244,7 @@ public static Container newContainer(ContainerId containerId, NodeId nodeId, container.setResource(resource); container.setPriority(priority); container.setContainerToken(containerToken); + container.setExecutionType(executionType); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java index e33c389..e6ecb7f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java @@ -160,7 +160,7 @@ private Container buildContainer(DistSchedulerParams appParams, containerTokenIdentifier); Container container = BuilderUtils.newContainer( cId, nodeId, nodeId.getHost() + ":" + webpagePort, - capability, rr.getPriority(), containerToken); + capability, rr.getPriority(), containerToken, containerTokenIdentifier.getExecutionType()); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index a5ca2d8..a5b3a49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -600,7 +600,7 @@ private Container createContainer(FiCaSchedulerNode node, Resource capability, // Create the container return BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, null); + .getHttpAddress(), capability, priority, null, null); } private ContainerAllocation handleNewContainerAllocation( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 82d665c..57d0581 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -473,7 +473,7 @@ public Container createContainer( // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, null); + .getHttpAddress(), capability, priority, null, null); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 3e6225f..aada3fd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -712,7 +712,7 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, null); + .getHttpAddress(), capability, priority, null, null); // Allocate! diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index b4ebf92..5514f90 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -188,7 +188,8 @@ synchronized public StartContainersResponse startContainers( Container container = BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, - tokenId.getResource(), null, null // DKDC - Doesn't matter + tokenId.getResource(), null, null /* DKDC - Doesn't matter */, + tokenId.getExecutionType() ); ContainerStatus containerStatus = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index ed8d56f..6c8b5d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -34,17 +34,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -94,7 +84,7 @@ public void testReleaseWhileRunning() { Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, - "host:3465", resource, priority, null); + "host:3465", resource, priority, null, null); ConcurrentMap rmApps = spy(new ConcurrentHashMap()); RMApp rmApp = mock(RMApp.class); @@ -194,7 +184,7 @@ public void testExpireWhileRunning() { Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, - "host:3465", resource, priority, null); + "host:3465", resource, priority, null, null); ConcurrentMap appMap = new ConcurrentHashMap<>(); RMApp rmApp = mock(RMApp.class);