diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 440779c..b40a40a 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -171,6 +172,16 @@ public String getNodeManagerVersion() { public Set getNodeLabels() { return RMNodeLabelsManager.EMPTY_STRING_SET; } + + @Override + public ResourceUtilization getContainersUtilization() { + return null; + } + + @Override + public ResourceUtilization getNodeUtilization() { + return null; + } } public static RMNode newNodeInfo(String rackName, String hostName, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index a6633ae..3dfd9f1 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -158,4 +159,14 @@ public String getNodeManagerVersion() { public Set getNodeLabels() { return RMNodeLabelsManager.EMPTY_STRING_SET; } + + @Override + public ResourceUtilization getContainersUtilization() { + return node.getContainersUtilization(); + } + + @Override + public ResourceUtilization getNodeUtilization() { + return node.getNodeUtilization(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 3c2c09b..4efb815 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -458,9 +458,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) // 4. Send status to RMNode, saving the latest response. RMNodeStatusEvent nodeStatusEvent = - new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(), - remoteNodeStatus.getContainersStatuses(), - remoteNodeStatus.getKeepAliveApplications(), nodeHeartBeatResponse); + new RMNodeStatusEvent(nodeId, remoteNodeStatus, nodeHeartBeatResponse); if (request.getLogAggregationReportsForApps() != null && !request.getLogAggregationReportsForApps().isEmpty()) { nodeStatusEvent.setLogAggregationReportsForApps(request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index 0386be6..371077d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; /** * Node managers information on available resources @@ -100,7 +101,19 @@ * @return the total available resource. */ public Resource getTotalCapability(); - + + /** + * the total resource utilization of the containers. + * @return the total resource utilization of the containers. + */ + public ResourceUtilization getContainersUtilization(); + + /** + * the total resource utilization of the node. + * @return the total resource utilization of the node. + */ + public ResourceUtilization getNodeUtilization(); + /** * The rack name for this node manager. * @return the rack name. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index f182d02..e2dd5a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType; @@ -108,6 +109,11 @@ private long lastHealthReportTime; private String nodeManagerVersion; + /* Aggregated resource utilization for the containers. */ + private ResourceUtilization containersUtilization; + /* Resource utilization for the node. */ + private ResourceUtilization nodeUtilization; + private final ContainerAllocationExpirer containerAllocationExpirer; /* set of containers that have just launched */ private final Set launchedContainers = @@ -369,6 +375,49 @@ public String getNodeManagerVersion() { } @Override + public ResourceUtilization getContainersUtilization() { + this.readLock.lock(); + + try { + return this.containersUtilization; + } finally { + this.readLock.unlock(); + } + } + + public void setContainersUtilization( + ResourceUtilization containersUtilization) { + this.writeLock.lock(); + + try { + this.containersUtilization = containersUtilization; + } finally { + this.writeLock.unlock(); + } + } + + @Override + public ResourceUtilization getNodeUtilization() { + this.readLock.lock(); + + try { + return this.nodeUtilization; + } finally { + this.readLock.unlock(); + } + } + + public void setNodeUtilization(ResourceUtilization nodeUtilization) { + this.writeLock.lock(); + + try { + this.nodeUtilization = nodeUtilization; + } finally { + this.writeLock.unlock(); + } + } + + @Override public NodeState getState() { this.readLock.lock(); @@ -805,6 +854,8 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) { rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport()); rmNode.setLastHealthReportTime( remoteNodeHealthStatus.getLastHealthReportTime()); + rmNode.setContainersUtilization(statusEvent.getContainersUtilization()); + rmNode.setNodeUtilization(statusEvent.getNodeUtilization()); if (!remoteNodeHealthStatus.getIsNodeHealthy()) { LOG.info("Node " + rmNode.nodeId + " reported UNHEALTHY with details: " + remoteNodeHealthStatus.getHealthReport()); @@ -860,6 +911,8 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) { rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport()); rmNode.setLastHealthReportTime( remoteNodeHealthStatus.getLastHealthReportTime()); + rmNode.setContainersUtilization(statusEvent.getContainersUtilization()); + rmNode.setNodeUtilization(statusEvent.getNodeUtilization()); if (remoteNodeHealthStatus.getIsNodeHealthy()) { rmNode.context.getDispatcher().getEventHandler().handle( new NodeAddedSchedulerEvent(rmNode)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java index b95d7d3..69104ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java @@ -25,44 +25,35 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; public class RMNodeStatusEvent extends RMNodeEvent { - private final NodeHealthStatus nodeHealthStatus; - private final List containersCollection; + private final NodeStatus nodeStatus; private final NodeHeartbeatResponse latestResponse; - private final List keepAliveAppIds; private List logAggregationReportsForApps; - public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus, - List collection, List keepAliveAppIds, + public RMNodeStatusEvent(NodeId nodeId, NodeStatus nodeStatus, NodeHeartbeatResponse latestResponse) { - super(nodeId, RMNodeEventType.STATUS_UPDATE); - this.nodeHealthStatus = nodeHealthStatus; - this.containersCollection = collection; - this.keepAliveAppIds = keepAliveAppIds; - this.latestResponse = latestResponse; - this.logAggregationReportsForApps = null; + this(nodeId, nodeStatus, latestResponse, null); } - public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus, - List collection, List keepAliveAppIds, + public RMNodeStatusEvent(NodeId nodeId, NodeStatus nodeStatus, NodeHeartbeatResponse latestResponse, List logAggregationReportsForApps) { super(nodeId, RMNodeEventType.STATUS_UPDATE); - this.nodeHealthStatus = nodeHealthStatus; - this.containersCollection = collection; - this.keepAliveAppIds = keepAliveAppIds; + this.nodeStatus = nodeStatus; this.latestResponse = latestResponse; this.logAggregationReportsForApps = logAggregationReportsForApps; } public NodeHealthStatus getNodeHealthStatus() { - return this.nodeHealthStatus; + return this.nodeStatus.getNodeHealthStatus(); } public List getContainers() { - return this.containersCollection; + return this.nodeStatus.getContainersStatuses(); } public NodeHeartbeatResponse getLatestResponse() { @@ -70,7 +61,15 @@ public NodeHeartbeatResponse getLatestResponse() { } public List getKeepAliveAppIds() { - return this.keepAliveAppIds; + return this.nodeStatus.getKeepAliveApplications(); + } + + public ResourceUtilization getContainersUtilization() { + return this.nodeStatus.getContainersUtilization(); + } + + public ResourceUtilization getNodeUtilization() { + return this.nodeStatus.getNodeUtilization(); } public List getLogAggregationReportsForApps() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index f03663a..1faa1c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -58,6 +59,10 @@ private Resource totalResourceCapability; private RMContainer reservedContainer; private volatile int numContainers; + private ResourceUtilization containersUtilization = + ResourceUtilization.newInstance(0, 0, 0f); + private ResourceUtilization nodeUtilization = + ResourceUtilization.newInstance(0, 0, 0f); /* set of containers that are allocated containers */ @@ -308,4 +313,41 @@ public String getPartition() { return this.labels.iterator().next(); } } + + /** + * Set the resource utilization of the containers in the node. + * @param containersUtilization Resource utilization of the containers. + */ + public void setContainersUtilization( + ResourceUtilization containersUtilization) { + if (containersUtilization != null) { + this.containersUtilization = containersUtilization; + } + } + + /** + * Get the resource utilization of the containers in the node. + * @return Resource utilization of the containers. + */ + public ResourceUtilization getContainersUtilization() { + return this.containersUtilization; + } + + /** + * Set the resource utilization of the node. This includes the containers. + * @param containersUtilization Resource utilization of the node. + */ + public void setNodeUtilization(ResourceUtilization nodeUtilization) { + if (nodeUtilization != null) { + this.nodeUtilization = nodeUtilization; + } + } + + /** + * Get the resource utilization of the node. + * @return Resource utilization of the node. + */ + public ResourceUtilization getNodeUtilization() { + return this.nodeUtilization; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index b4b1383..73c48c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1013,6 +1013,10 @@ private synchronized void nodeUpdate(RMNode nm) { releaseResources); schedulerHealth.updateSchedulerReleaseCounts(releasedContainers); + // Updating node resource utilization + node.setContainersUtilization(nm.getContainersUtilization()); + node.setNodeUtilization(nm.getNodeUtilization()); + // Now node data structures are upto date and ready for scheduling. if(LOG.isDebugEnabled()) { LOG.debug("Node being looked for scheduling " + nm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 095fe28..9358337 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; @@ -108,11 +109,14 @@ public static Resource newAvailResource(Resource total, Resource used) { private long lastHealthReportTime; private NodeState state; private Set labels; + private ResourceUtilization containersUtilization; + private ResourceUtilization nodeUtilization; public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, long lastHealthReportTime, int cmdPort, String hostName, NodeState state, - Set labels) { + Set labels, ResourceUtilization containersUtilization, + ResourceUtilization nodeUtilization) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; this.httpAddress = httpAddress; @@ -124,6 +128,8 @@ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, this.hostName = hostName; this.state = state; this.labels = labels; + this.containersUtilization = containersUtilization; + this.nodeUtilization = nodeUtilization; } @Override @@ -227,6 +233,16 @@ public long getLastHealthReportTime() { } return CommonNodeLabelsManager.EMPTY_STRING_SET; } + + @Override + public ResourceUtilization getContainersUtilization() { + return this.containersUtilization; + } + + @Override + public ResourceUtilization getNodeUtilization() { + return this.nodeUtilization; + } }; private static RMNode buildRMNode(int rack, final Resource perNode, @@ -237,18 +253,19 @@ private static RMNode buildRMNode(int rack, final Resource perNode, private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr, Set labels) { return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123, - labels); + labels, null, null); } private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr, int hostnum, String hostName, int port) { return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port, - null); + null, null, null); } private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr, int hostnum, String hostName, int port, - Set labels) { + Set labels, ResourceUtilization containersUtilization, + ResourceUtilization nodeUtilization) { final String rackName = "rack"+ rack; final int nid = hostnum; final String nodeAddr = hostName + ":" + nid; @@ -260,7 +277,8 @@ private static RMNode buildRMNode(int rack, final Resource perNode, final String httpAddress = httpAddr; String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, - rackName, healthReport, 0, nid, hostName, state, labels); + rackName, healthReport, 0, nid, hostName, state, labels, + containersUtilization, nodeUtilization); } public static RMNode nodeInfo(int rack, final Resource perNode, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 4964c59..62d6183 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent; @@ -537,8 +538,9 @@ public void testUpdateHeartbeatResponseForAppLifeCycle() { statusList.add(status); NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "", System.currentTimeMillis()); - node.handle(new RMNodeStatusEvent(nodeId, nodeHealth, - statusList, null, null)); + NodeStatus nodeStatus = NodeStatus.newInstance(nodeId, 1, statusList, null, + nodeHealth, null, null); + node.handle(new RMNodeStatusEvent(nodeId, nodeStatus, null)); Assert.assertEquals(1, node.getRunningApps().size()); @@ -571,8 +573,9 @@ private RMNodeImpl getUnhealthyNode() { RMNodeImpl node = getRunningNode(); NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick", System.currentTimeMillis()); - node.handle(new RMNodeStatusEvent(node.getNodeID(), status, - new ArrayList(), null, null)); + NodeStatus nodeStatus = NodeStatus.newInstance(node.getNodeID(), 1, + new ArrayList(), null, status, null, null); + node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null)); Assert.assertEquals(NodeState.UNHEALTHY, node.getState()); return node; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java index fccfa19..f762d31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; @@ -163,9 +164,11 @@ public void testLogAggregationStatus() throws Exception { LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode1_1); node1ReportForApp.add(report1); - node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus - .newInstance(true, null, 0), new ArrayList(), null, - null, node1ReportForApp)); + NodeStatus nodeStatus1 = NodeStatus.newInstance(node1.getNodeID(), 0, + new ArrayList(), null, + NodeHealthStatus.newInstance(true, null, 0), null, null); + node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, + node1ReportForApp)); List node2ReportForApp = new ArrayList(); @@ -175,9 +178,11 @@ public void testLogAggregationStatus() throws Exception { LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode2_1); node2ReportForApp.add(report2); - node2.handle(new RMNodeStatusEvent(node2.getNodeID(), NodeHealthStatus - .newInstance(true, null, 0), new ArrayList(), null, - null, node2ReportForApp)); + NodeStatus nodeStatus2 = NodeStatus.newInstance(node2.getNodeID(), 0, + new ArrayList(), null, + NodeHealthStatus.newInstance(true, null, 0), null, null); + node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null, + node2ReportForApp)); // node1 and node2 has updated its log aggregation status // verify that the log aggregation status for node1, node2 // has been changed @@ -213,9 +218,8 @@ public void testLogAggregationStatus() throws Exception { LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode1_2); node1ReportForApp2.add(report1_2); - node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus - .newInstance(true, null, 0), new ArrayList(), null, - null, node1ReportForApp2)); + node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, + node1ReportForApp2)); // verify that the log aggregation status for node1 // has been changed @@ -282,9 +286,8 @@ public void testLogAggregationStatus() throws Exception { LogAggregationStatus.SUCCEEDED, "")); // For every logAggregationReport cached in memory, we can only save at most // 10 diagnostic messages/failure messages - node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus - .newInstance(true, null, 0), new ArrayList(), null, - null, node1ReportForApp3)); + node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, + node1ReportForApp3)); logAggregationStatus = rmApp.getLogAggregationReportsForApp(); Assert.assertEquals(2, logAggregationStatus.size()); @@ -327,9 +330,8 @@ public void testLogAggregationStatus() throws Exception { LogAggregationStatus.FAILED, ""); node2ReportForApp2.add(report2_2); node2ReportForApp2.add(report2_3); - node2.handle(new RMNodeStatusEvent(node2.getNodeID(), NodeHealthStatus - .newInstance(true, null, 0), new ArrayList(), null, - null, node2ReportForApp2)); + node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null, + node2ReportForApp2)); Assert.assertEquals(LogAggregationStatus.FAILED, rmApp.getLogAggregationStatusForAppReport()); logAggregationStatus = rmApp.getLogAggregationReportsForApp(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index ec20bba..dc40052 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -142,8 +143,9 @@ public void testNodesDefaultWithUnHealthyNode() throws JSONException, .get(nm3.getNodeId()); NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false, "test health report", System.currentTimeMillis()); - node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeHealth, - new ArrayList(), null, null)); + NodeStatus nodeStatus = NodeStatus.newInstance(nm3.getNodeId(), 1, + new ArrayList(), null, nodeHealth, null, null); + node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeStatus, null)); rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY); ClientResponse response =