diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 4ac8e07..9478d34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -125,18 +125,6 @@ void setPriority(Priority priority); /** - * Get the current ContainerState of the container. - * @return current ContainerState of the container - */ - @Public - @Stable - ContainerState getState(); - - @Private - @Unstable - void setState(ContainerState state); - - /** * Get the ContainerToken for the container. * @return ContainerToken for the container */ @@ -147,16 +135,4 @@ @Private @Unstable void setContainerToken(ContainerToken containerToken); - - /** - * Get the ContainerStatus of the container. - * @return ContainerStatus of the container - */ - @Public - @Stable - ContainerStatus getContainerStatus(); - - @Private - @Unstable - void setContainerStatus(ContainerStatus containerStatus); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index 92a710a..7c4d1f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -94,11 +94,6 @@ private void mergeLocalToBuilder() { builder.getContainerToken())) { builder.setContainerToken(convertToProtoFormat(this.containerToken)); } - if (this.containerStatus != null - && !((ContainerStatusPBImpl) this.containerStatus).getProto().equals( - builder.getContainerStatus())) { - builder.setContainerStatus(convertToProtoFormat(this.containerStatus)); - } } private void mergeLocalToProto() { @@ -115,27 +110,8 @@ private void maybeInitBuilder() { } viaProto = false; } - - - @Override - public ContainerState getState() { - ContainerProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasState()) { - return null; - } - return convertFromProtoFormat(p.getState()); - } @Override - public void setState(ContainerState state) { - maybeInitBuilder(); - if (state == null) { - builder.clearState(); - return; - } - builder.setState(convertToProtoFormat(state)); - } - @Override public ContainerId getId() { ContainerProtoOrBuilder p = viaProto ? proto : builder; if (this.containerId != null) { @@ -260,35 +236,6 @@ public void setContainerToken(ContainerToken containerToken) { this.containerToken = containerToken; } - @Override - public ContainerStatus getContainerStatus() { - ContainerProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerStatus != null) { - return this.containerStatus; - } - if (!p.hasContainerStatus()) { - return null; - } - this.containerStatus = convertFromProtoFormat(p.getContainerStatus()); - return this.containerStatus; - } - - @Override - public void setContainerStatus(ContainerStatus containerStatus) { - maybeInitBuilder(); - if (containerStatus == null) - builder.clearContainerStatus(); - this.containerStatus = containerStatus; - } - - private ContainerStateProto convertToProtoFormat(ContainerState e) { - return ProtoUtils.convertToProtoFormat(e); - } - - private ContainerState convertFromProtoFormat(ContainerStateProto e) { - return ProtoUtils.convertFromProtoFormat(e); - } - private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { return new ContainerIdPBImpl(p); } @@ -329,14 +276,6 @@ private TokenProto convertToProtoFormat(ContainerToken t) { return ((ContainerTokenPBImpl)t).getProto(); } - private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) { - return new ContainerStatusPBImpl(p); - } - - private ContainerStatusProto convertToProtoFormat(ContainerStatus t) { - return ((ContainerStatusPBImpl)t).getProto(); - } - public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Container: ["); @@ -345,9 +284,7 @@ public String toString() { sb.append("NodeHttpAddress: ").append(getNodeHttpAddress()).append(", "); sb.append("Resource: ").append(getResource()).append(", "); sb.append("Priority: ").append(getPriority()).append(", "); - sb.append("State: ").append(getState()).append(", "); sb.append("Token: ").append(getContainerToken()).append(", "); - sb.append("Status: ").append(getContainerStatus()); sb.append("]"); return sb.toString(); } @@ -357,16 +294,7 @@ public String toString() { public int compareTo(Container other) { if (this.getId().compareTo(other.getId()) == 0) { if (this.getNodeId().compareTo(other.getNodeId()) == 0) { - if (this.getResource().compareTo(other.getResource()) == 0) { - if (this.getState().compareTo(other.getState()) == 0) { - //ContainerToken - return this.getState().compareTo(other.getState()); - } else { - return this.getState().compareTo(other.getState()); - } - } else { - return this.getResource().compareTo(other.getResource()); - } + return this.getResource().compareTo(other.getResource()); } else { return this.getNodeId().compareTo(other.getNodeId()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 4d0ca0f..83aac74 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -67,9 +67,7 @@ message ContainerProto { optional string node_http_address = 3; optional ResourceProto resource = 4; optional PriorityProto priority = 5; - optional ContainerStateProto state = 6; - optional hadoop.common.TokenProto container_token = 7; - optional ContainerStatusProto container_status = 8; + optional hadoop.common.TokenProto container_token = 6; } enum YarnApplicationStateProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 1e65a9a..2bd33c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -608,7 +608,6 @@ public void onContainersAllocated(List allocatedContainers) { + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() - + ", containerState" + allocatedContainer.getState() + ", containerResourceMemory" + allocatedContainer.getResource().getMemory()); // + ", containerToken" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 502f1dd..b2aabc8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -246,11 +246,6 @@ public static Container newContainer(ContainerId containerId, container.setNodeHttpAddress(nodeHttpAddress); container.setResource(resource); container.setPriority(priority); - container.setState(ContainerState.NEW); - ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class); - containerStatus.setContainerId(containerId); - containerStatus.setState(ContainerState.NEW); - container.setContainerStatus(containerStatus); container.setContainerToken(containerToken); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 69ac601..d44fd3f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -288,14 +288,9 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { public void transition(RMContainerImpl container, RMContainerEvent event) { RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event; - // Update container-status for diagnostics. Today we completely - // replace it on finish. We may just need to update diagnostics. - container.container.setContainerStatus(finishedEvent - .getRemoteContainerStatus()); - // Inform AppAttempt container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( - container.appAttemptId, container.container.getContainerStatus())); + container.appAttemptId, finishedEvent.getRemoteContainerStatus())); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index c85f233..a2f59ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; @@ -71,11 +70,11 @@ void setResourceTrackerService(ResourceTrackerService resourceTracker) { this.resourceTracker = resourceTracker; } - public void containerStatus(Container container) throws Exception { + public void containerStatus(ContainerStatus containerStatus) throws Exception { Map> conts = new HashMap>(); - conts.put(container.getId().getApplicationAttemptId().getApplicationId(), - Arrays.asList(new ContainerStatus[] { container.getContainerStatus() })); + conts.put(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(), + Arrays.asList(new ContainerStatus[] { containerStatus })); nodeHeartbeat(conts, true); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 170938d..8af339d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -76,6 +76,9 @@ final Map> containers = new HashMap>(); + final Map containerStatusMap = + new HashMap(); + public NodeManager(String hostName, int containerManagerPort, int httpPort, String rackName, Resource capability, ResourceTrackerService resourceTrackerService, RMContext rmContext) @@ -137,7 +140,7 @@ public Resource getUsed() { List containerStatuses = new ArrayList(); for (List appContainers : containers.values()) { for (Container container : appContainers) { - containerStatuses.add(container.getContainerStatus()); + containerStatuses.add(containerStatusMap.get(container)); } } return containerStatuses; @@ -189,8 +192,11 @@ synchronized public StartContainerResponse startContainer( null, null // DKDC - Doesn't matter ); + ContainerStatus containerStatus = + BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW, + "", -1000); applicationContainers.add(container); - + containerStatusMap.put(container, containerStatus); Resources.subtractFrom(available, containerLaunchContext.getResource()); Resources.addTo(used, containerLaunchContext.getResource()); @@ -223,7 +229,9 @@ synchronized public StopContainerResponse stopContainer(StopContainerRequest req List applicationContainers = containers.get(applicationId); for (Container c : applicationContainers) { if (c.getId().compareTo(containerID) == 0) { - c.setState(ContainerState.COMPLETE); + ContainerStatus containerStatus = containerStatusMap.get(c); + containerStatus.setState(ContainerState.COMPLETE); + containerStatusMap.put(c, containerStatus); } } @@ -277,8 +285,8 @@ synchronized public GetContainerStatusResponse getContainerStatus(GetContainerSt } GetContainerStatusResponse response = recordFactory.newRecordInstance(GetContainerStatusResponse.class); - if (container != null && container.getContainerStatus() != null) { - response.setStatus(container.getContainerStatus()); + if (container != null && containerStatusMap.get(container).getState() != null) { + response.setStatus(containerStatusMap.get(container)); } return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 4780ec3..88bd5a6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -139,8 +139,9 @@ public void test() throws Exception { Container c1 = allocated1.get(0); Assert.assertEquals(GB, c1.getResource().getMemory()); - c1.setState(ContainerState.COMPLETE); - nm1.containerStatus(c1); + ContainerStatus containerStatus = BuilderUtils.newContainerStatus( + c1.getId(), ContainerState.COMPLETE, "", 0); + nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) {