diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java index 0e55529..1cfe31f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java @@ -123,11 +123,17 @@ public void testRMNMInfo() throws Exception { Assert.assertNotNull(n.get("NodeManagerVersion")); Assert.assertNotNull(n.get("NumContainers")); Assert.assertEquals( - n.get("NodeId") + ": Unexpected number of used containers", - 0, n.get("NumContainers").asInt()); + n.get("NodeId") + ": Unexpected number of used guaranteed containers", + 0, n.get("NumContainers").asInt()); + Assert.assertEquals(n.get("NodeId") + + ": Unexpected number of used opportunistic containers", + 0, n.get("NumOpportunisticContainers").asInt()); Assert.assertEquals( n.get("NodeId") + ": Unexpected amount of used memory", 0, n.get("UsedMemoryMB").asInt()); + Assert.assertEquals( + n.get("NodeId") + ": Unexpected amount of used opportunistic memory", + 0, n.get("UsedOpportunisticMemoryMB").asInt()); Assert.assertNotNull(n.get("AvailableMemoryMB")); } } @@ -160,6 +166,8 @@ public void testRMNMInfoMissmatch() throws Exception { Assert.assertNotNull(n.get("NodeManagerVersion")); Assert.assertNull(n.get("NumContainers")); Assert.assertNull(n.get("UsedMemoryMB")); + Assert.assertNull(n.get("NumOpportunisticContainers")); + Assert.assertNull(n.get("UsedOpportunisticMemoryMB")); Assert.assertNull(n.get("AvailableMemoryMB")); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java index 885a3b4..4059516 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java @@ -53,7 +53,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, int numContainers, String healthReport, long lastHealthReportTime) { return newInstance(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, null); + capability, numContainers, healthReport, lastHealthReportTime, + null, null, 0); } @Private @@ -61,7 +62,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, int numContainers, String healthReport, long lastHealthReportTime, - Set nodeLabels) { + Set nodeLabels, + Resource opportunisticUsed, int numOpportunisticContainers) { NodeReport nodeReport = Records.newRecord(NodeReport.class); nodeReport.setNodeId(nodeId); nodeReport.setNodeState(nodeState); @@ -73,6 +75,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); + nodeReport.setUsedOpportunisticResource(opportunisticUsed); + nodeReport.setNumOpportunisticContainers(numOpportunisticContainers); return nodeReport; } @@ -125,8 +129,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, public abstract void setRackName(String rackName); /** - * Get used Resource on the node. - * @return used Resource on the node + * Get used guaranteed Resource on the node. + * @return used guaranteed Resource on the node */ @Public @Stable @@ -135,7 +139,19 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, @Private @Unstable public abstract void setUsed(Resource used); - + + /** + * Get used opportunistic Resource on the node. + * @return used opportunistic Resource on the node + */ + @Public + @Stable + public abstract Resource getUsedOpportunisticResource(); + + @Private + @Unstable + public abstract void setUsedOpportunisticResource(Resource used); + /** * Get the total Resource on the node. * @return total Resource on the node @@ -149,8 +165,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, public abstract void setCapability(Resource capability); /** - * Get the number of allocated containers on the node. - * @return number of allocated containers on the node + * Get the number of allocated guaranteed containers on the node. + * @return number of allocated guaranteed containers on the node */ @Private @Unstable @@ -159,7 +175,18 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, @Private @Unstable public abstract void setNumContainers(int numContainers); - + + /** + * Get the number of allocated opportunistic containers on the node. + * @return number of allocated opportunistic containers on the node + */ + @Private + @Unstable + public abstract int getNumOpportunisticContainers(); + + @Private + @Unstable + public abstract void setNumOpportunisticContainers(int numContainers); /** * Get the diagnostic health report of the node. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index e69c07b..6fe5c92 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -345,6 +345,8 @@ message NodeReportProto { repeated string node_labels = 10; optional ResourceUtilizationProto containers_utilization = 11; optional ResourceUtilizationProto node_utilization = 12; + optional int32 numOpportunisticContainers = 13; + optional ResourceProto usedOpportunisticResource = 14; } message NodeIdToLabelsProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index f4005e9..4510e07 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -657,7 +657,7 @@ public YarnClusterMetrics createFakeYarnClusterMetrics() { NodeId nodeId = NodeId.newInstance("localhost", 0); NodeReport report = NodeReport.newInstance(nodeId, NodeState.RUNNING, "localhost", - "rack1", null, null, 4, null, 1000l, null); + "rack1", null, null, 4, null, 1000l, null, null, 0); List reports = new ArrayList(); reports.add(report); return reports; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 84cfb0a..32d5f2d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1989,10 +1989,11 @@ private void verifyUsageInfo(YarnCLI cli) throws Exception { // ordered nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y"); } - NodeReport nodeReport = NodeReport.newInstance(NodeId - .newInstance("host" + i, 0), state, "host" + 1 + ":8888", - "rack1", Records.newRecord(Resource.class), Records - .newRecord(Resource.class), 0, "", 0, nodeLabels); + NodeReport nodeReport = NodeReport.newInstance( + NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888", + "rack1", Records.newRecord(Resource.class), + Records.newRecord(Resource.class), 0, "", 0, nodeLabels, + Records.newRecord(Resource.class), 0); if (!emptyResourceUtilization) { ResourceUtilization containersUtilization = ResourceUtilization .newInstance(1024, 2048, 4); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java index 0d205e9..03e1f65 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java @@ -45,6 +45,7 @@ private boolean viaProto = false; private NodeId nodeId; private Resource used; + private Resource opportunisticResourceUsed; private Resource capability; private ResourceUtilization containersUtilization = null; private ResourceUtilization nodeUtilization = null; @@ -114,6 +115,13 @@ public int getNumContainers() { } @Override + public int getNumOpportunisticContainers() { + NodeReportProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasNumOpportunisticContainers()) ? + p.getNumOpportunisticContainers() : 0; + } + + @Override public String getRackName() { NodeReportProtoOrBuilder p = viaProto ? proto : builder; return (p.hasRackName()) ? p.getRackName() : null; @@ -134,6 +142,21 @@ public Resource getUsed() { } @Override + public Resource getUsedOpportunisticResource() { + if (this.opportunisticResourceUsed != null) { + return this.opportunisticResourceUsed; + } + + NodeReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasUsedOpportunisticResource()) { + return null; + } + this.opportunisticResourceUsed = + convertFromProtoFormat(p.getUsedOpportunisticResource()); + return this.opportunisticResourceUsed; + } + + @Override public NodeId getNodeId() { if (this.nodeId != null) { return this.nodeId; @@ -203,6 +226,17 @@ public void setNumContainers(int numContainers) { } @Override + public void setNumOpportunisticContainers(int numContainers) { + maybeInitBuilder(); + if (numContainers == 0) { + builder.clearNumOpportunisticContainers(); + return; + } + builder.setNumOpportunisticContainers(numContainers); + + } + + @Override public void setRackName(String rackName) { maybeInitBuilder(); if (rackName == null) { @@ -220,6 +254,15 @@ public void setUsed(Resource used) { this.used = used; } + @Override + public void setUsedOpportunisticResource(Resource opportunisticUsed) { + maybeInitBuilder(); + if (opportunisticUsed == null) { + builder.clearUsedOpportunisticResource(); + } + this.opportunisticResourceUsed = opportunisticUsed; + } + public NodeReportProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); @@ -256,6 +299,10 @@ private void mergeLocalToBuilder() { if (this.used != null) { builder.setUsed(convertToProtoFormat(this.used)); } + if (this.opportunisticResourceUsed != null) { + builder.setUsedOpportunisticResource( + convertToProtoFormat(this.opportunisticResourceUsed)); + } if (this.capability != null) { builder.setCapability(convertToProtoFormat(this.capability)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index fec2681..3b1ccde 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -185,23 +185,20 @@ public static NodeId newNodeId(String host, int port) { public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime) { + int numContainers, Resource opportunisticResourceUsed, + int numOpportunisticContainer, String healthReport, + long lastHealthReportTime, Set nodeLabels) { return newNodeReport(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, null); - } - - public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime, - Set nodeLabels) { - return newNodeReport(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, + capability, numContainers, opportunisticResourceUsed, + numOpportunisticContainer, healthReport, lastHealthReportTime, nodeLabels, null, null); } public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime, + String httpAddress, String rackName, Resource guaranteedResourceUsed, + Resource capability, int numGuaranteedContainers, + Resource opportunisticResourceUsed, int numOpportunisticContainers, + String healthReport, long lastHealthReportTime, Set nodeLabels, ResourceUtilization containersUtilization, ResourceUtilization nodeUtilization) { NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class); @@ -209,9 +206,11 @@ public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, nodeReport.setNodeState(nodeState); nodeReport.setHttpAddress(httpAddress); nodeReport.setRackName(rackName); - nodeReport.setUsed(used); + nodeReport.setUsed(guaranteedResourceUsed); nodeReport.setCapability(capability); - nodeReport.setNumContainers(numContainers); + nodeReport.setNumContainers(numGuaranteedContainers); + nodeReport.setUsedOpportunisticResource(opportunisticResourceUsed); + nodeReport.setNumOpportunisticContainers(numOpportunisticContainers); nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index e9c6eac..55249a4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1027,17 +1027,26 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); - Resource used = BuilderUtils.newResource(0, 0); - int numContainers = 0; + Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0); + int numGuaranteedContainers = 0; + Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0); + int numOpportunisticContainers = 0; if (schedulerNodeReport != null) { - used = schedulerNodeReport.getUsedResource(); - numContainers = schedulerNodeReport.getNumContainers(); + guaranteedResourceUsed = schedulerNodeReport.getUsedGuaranteedResource(); + opportunisticResourceUsed = + schedulerNodeReport.getUsedOpportunisticResource(); + numGuaranteedContainers = + schedulerNodeReport.getNumGuaranteedContainers(); + numOpportunisticContainers = + schedulerNodeReport.getNumOpportunisticContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), - rmNode.getHttpAddress(), rmNode.getRackName(), used, - rmNode.getTotalCapability(), numContainers, + rmNode.getHttpAddress(), rmNode.getRackName(), + guaranteedResourceUsed, + rmNode.getTotalCapability(), numGuaranteedContainers, + opportunisticResourceUsed, numOpportunisticContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(), rmNode.getNodeUtilization()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 9774a1a..244008e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -332,17 +332,27 @@ private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) { for(RMNode rmNode: updatedNodes) { SchedulerNodeReport schedulerNodeReport = getScheduler().getNodeReport(rmNode.getNodeID()); - Resource used = BuilderUtils.newResource(0, 0); - int numContainers = 0; + Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0); + int numGuaranteedContainers = 0; + Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0); + int numOpportunisticContainers = 0; if (schedulerNodeReport != null) { - used = schedulerNodeReport.getUsedResource(); - numContainers = schedulerNodeReport.getNumContainers(); + opportunisticResourceUsed = + schedulerNodeReport.getUsedOpportunisticResource(); + guaranteedResourceUsed = + schedulerNodeReport.getUsedGuaranteedResource(); + numGuaranteedContainers = + schedulerNodeReport.getNumGuaranteedContainers(); + numOpportunisticContainers = + schedulerNodeReport.getNumOpportunisticContainers(); } NodeId nodeId = rmNode.getNodeID(); NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), - rmNode.getHttpAddress(), rmNode.getRackName(), used, - rmNode.getTotalCapability(), numContainers, + rmNode.getHttpAddress(), rmNode.getRackName(), + guaranteedResourceUsed, rmNode.getTotalCapability(), + numGuaranteedContainers, opportunisticResourceUsed, + numOpportunisticContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java index 1b7ddd3..f3fdbeb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java @@ -79,27 +79,32 @@ public String getLiveNodeManagers() { List nodesInfo = new ArrayList(); for (final RMNode ni : nodes) { - SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID()); - InfoMap info = new InfoMap(); - info.put("HostName", ni.getHostName()); - info.put("Rack", ni.getRackName()); - info.put("State", ni.getState().toString()); - info.put("NodeId", ni.getNodeID()); - info.put("NodeHTTPAddress", ni.getHttpAddress()); - info.put("LastHealthUpdate", - ni.getLastHealthReportTime()); - info.put("HealthReport", - ni.getHealthReport()); - info.put("NodeManagerVersion", - ni.getNodeManagerVersion()); - if(report != null) { - info.put("NumContainers", report.getNumContainers()); - info.put("UsedMemoryMB", report.getUsedResource().getMemorySize()); - info.put("AvailableMemoryMB", - report.getAvailableResource().getMemorySize()); - } + SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID()); + InfoMap info = new InfoMap(); + info.put("HostName", ni.getHostName()); + info.put("Rack", ni.getRackName()); + info.put("State", ni.getState().toString()); + info.put("NodeId", ni.getNodeID()); + info.put("NodeHTTPAddress", ni.getHttpAddress()); + info.put("LastHealthUpdate", + ni.getLastHealthReportTime()); + info.put("HealthReport", + ni.getHealthReport()); + info.put("NodeManagerVersion", + ni.getNodeManagerVersion()); + if(report != null) { + info.put("NumContainers", report.getNumGuaranteedContainers()); + info.put("NumOpportunisticContainers", + report.getNumOpportunisticContainers()); + info.put("UsedMemoryMB", + report.getUsedGuaranteedResource().getMemorySize()); + info.put("UsedOpportunisticMemoryMB", + report.getUsedOpportunisticResource().getMemorySize()); + info.put("AvailableMemoryMB", + report.getAvailableGuaranteedResource().getMemorySize()); + } - nodesInfo.add(info); + nodesInfo.add(info); } return JSON.toString(nodesInfo); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 01d2ce2..bb1d18c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -139,7 +139,7 @@ /** Physical resources in the node. */ private volatile Resource physicalResource; - /* Container Queue Information for the node.. Used by Distributed Scheduler */ + /* Container Queue Information for the node..*/ private OpportunisticContainersStatus opportunisticContainersStatus; private final ContainerAllocationExpirer containerAllocationExpirer; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java index ea30d78..5b134d6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java @@ -28,34 +28,54 @@ @Private @Stable public class SchedulerNodeReport { - private final Resource used; - private final Resource avail; - private final int num; + private final Resource guaranteedResourceUsage; + private final Resource opportunisticResourceUsage; + private final Resource guaranteedResourceAvail; + private final int numOfGuaranteedContainers; + private final int numOfOpportunisticContainers; public SchedulerNodeReport(SchedulerNode node) { - this.used = node.getAllocatedResource(); - this.avail = node.getUnallocatedResource(); - this.num = node.getNumGuaranteedContainers(); + this.guaranteedResourceUsage = node.getAllocatedResource(); + this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated(); + this.guaranteedResourceAvail = node.getUnallocatedResource(); + this.numOfGuaranteedContainers = node.getNumGuaranteedContainers(); + this.numOfOpportunisticContainers = node.getNumOpportunisticContainers(); } /** * @return the amount of resources currently used by the node. */ - public Resource getUsedResource() { - return used; + public Resource getUsedGuaranteedResource() { + return guaranteedResourceUsage; } /** - * @return the amount of resources currently available on the node + * @return the amount of opportunistic resources currently used by the node. */ - public Resource getAvailableResource() { - return avail; + public Resource getUsedOpportunisticResource() { + return opportunisticResourceUsage; } /** - * @return the number of containers currently running on this node. + * @return the amount of guaranteed resources currently available on the node */ - public int getNumContainers() { - return num; + public Resource getAvailableGuaranteedResource() { + return guaranteedResourceAvail; + } + + /** + * @return the number of guaranteed containers currently running on + * this node. + */ + public int getNumGuaranteedContainers() { + return numOfGuaranteedContainers; + } + + /** + * @return the number of opportunistic containers currently running on + * this node. + */ + public int getNumOpportunisticContainers() { + return numOfOpportunisticContainers; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java index 1752546..c5a0ce7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java @@ -79,10 +79,13 @@ public FifoSchedulerInfo(final ResourceManager rm) { for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); - this.usedNodeCapacity += report.getUsedResource().getMemorySize(); - this.availNodeCapacity += report.getAvailableResource().getMemorySize(); + this.usedNodeCapacity += + report.getUsedGuaranteedResource().getMemorySize(); + this.availNodeCapacity += + report.getAvailableGuaranteedResource().getMemorySize(); this.totalNodeCapacity += ni.getTotalCapability().getMemorySize(); - this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); + this.numContainers += + fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 2530c8e..04e2bef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -69,12 +69,14 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.usedMemoryMB = 0; this.availMemoryMB = 0; if (report != null) { - this.numContainers = report.getNumContainers(); - this.usedMemoryMB = report.getUsedResource().getMemorySize(); - this.availMemoryMB = report.getAvailableResource().getMemorySize(); - this.usedVirtualCores = report.getUsedResource().getVirtualCores(); + this.numContainers = report.getNumGuaranteedContainers(); + this.usedMemoryMB = report.getUsedGuaranteedResource().getMemorySize(); + this.availMemoryMB = + report.getAvailableGuaranteedResource().getMemorySize(); + this.usedVirtualCores = + report.getUsedGuaranteedResource().getVirtualCores(); this.availableVirtualCores = - report.getAvailableResource().getVirtualCores(); + report.getAvailableGuaranteedResource().getVirtualCores(); } this.id = id.toString(); this.rack = ni.getRackName(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index ee974e3..4de3c86 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -225,10 +225,10 @@ synchronized public void checkResourceUsage() { LOG.info("Checking resource usage for " + containerManagerAddress); Assert.assertEquals(available.getMemorySize(), resourceManager.getResourceScheduler().getNodeReport( - this.nodeId).getAvailableResource().getMemorySize()); + this.nodeId).getAvailableGuaranteedResource().getMemorySize()); Assert.assertEquals(used.getMemorySize(), resourceManager.getResourceScheduler().getNodeReport( - this.nodeId).getUsedResource().getMemorySize()); + this.nodeId).getUsedGuaranteedResource().getMemorySize()); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index cad0151..55648f1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -176,8 +176,10 @@ public void testApplicationPriorityAllocation() throws Exception { // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(15 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); @@ -189,8 +191,10 @@ public void testApplicationPriorityAllocation() throws Exception { // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // get scheduler CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -210,8 +214,10 @@ public void testApplicationPriorityAllocation() throws Exception { // check node report, 12 GB used and 4 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(12 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // send updated request for App1 am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList()); @@ -226,8 +232,10 @@ public void testApplicationPriorityAllocation() throws Exception { // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } @@ -261,8 +269,10 @@ public void testPriorityWithPendingApplications() throws Exception { // check node report, 8 GB used (1 AM and 7 containers) and 0 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(8 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 7 Priority appPriority2 = Priority.newInstance(7); @@ -288,8 +298,10 @@ public void testPriorityWithPendingApplications() throws Exception { // check node report, 1 GB used and 7 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(7 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } @@ -486,8 +498,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(15 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); @@ -499,8 +513,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // get scheduler CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -522,8 +538,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // check node report, 12 GB used and 4 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(12 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers App1 am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList()); @@ -535,8 +553,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() Assert.assertEquals(2, allocated2.size()); // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // kill 1 more counter = 0; @@ -552,8 +572,10 @@ public void testApplicationPriorityAllocationWithChangeInPriority() // check node report, 14 GB used and 2 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(14 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Change the priority of App1 to 3 (lowest) Priority appPriority3 = Priority.newInstance(3); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index fe50eb1..bb56023 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -1265,8 +1265,10 @@ public void testResourceOverCommit() throws Exception { SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); @@ -1287,8 +1289,10 @@ public void testResourceOverCommit() throws Exception { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); @@ -1307,7 +1311,7 @@ public void testResourceOverCommit() throws Exception { waitCount = 0; while (waitCount++ != 20) { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - if (report_nm1.getAvailableResource().getMemorySize() != 0) { + if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) { break; } LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried " @@ -1316,8 +1320,10 @@ public void testResourceOverCommit() throws Exception { } // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(-2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( @@ -1333,9 +1339,11 @@ public void testResourceOverCommit() throws Exception { Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); // As container return 2 GB back, the available resource becomes 0 again. - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Verify no NPE is trigger in schedule after resource is updated. am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1); @@ -2835,8 +2843,10 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report - Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(9 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index c56be29..4a04a39 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -1947,15 +1947,17 @@ public RMNodeLabelsManager createNodeLabelManager() { SchedulerNodeReport reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize()); Assert.assertEquals(5 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(5 * GB, + reportNm1.getAvailableGuaranteedResource().getMemorySize()); SchedulerNodeReport reportNm2 = rm1.getResourceScheduler() .getNodeReport(nm2.getNodeId()); - Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + reportNm2.getUsedGuaranteedResource().getMemorySize()); Assert.assertEquals(10 * GB, - reportNm2.getAvailableResource().getMemorySize()); + reportNm2.getAvailableGuaranteedResource().getMemorySize()); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a"); assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB()); @@ -2047,15 +2049,17 @@ public RMNodeLabelsManager createNodeLabelManager() { SchedulerNodeReport reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize()); + Assert.assertEquals(3 * GB, + reportNm1.getUsedGuaranteedResource().getMemorySize()); Assert.assertEquals(7 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getAvailableGuaranteedResource().getMemorySize()); SchedulerNodeReport reportNm2 = rm1.getResourceScheduler() .getNodeReport(nm2.getNodeId()); - Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize()); + Assert.assertEquals(1 * GB, + reportNm2.getUsedGuaranteedResource().getMemorySize()); Assert.assertEquals(9 * GB, - reportNm2.getAvailableResource().getMemorySize()); + reportNm2.getAvailableGuaranteedResource().getMemorySize()); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a"); double delta = 0.0001; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 63da0c3..76dfb35 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -697,7 +697,8 @@ public void testFifoScheduling() throws Exception { am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 @@ -707,7 +708,8 @@ public void testFifoScheduling() throws Exception { am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); - Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getUsedGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); @@ -743,11 +745,15 @@ public void testFifoScheduling() throws Exception { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getAvailableGuaranteedResource().getMemorySize()); - Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize()); + Assert.assertEquals(6 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getUsedGuaranteedResource().getMemorySize()); Container c1 = allocated1.get(0); Assert.assertEquals(GB, c1.getResource().getMemorySize()); @@ -765,7 +771,8 @@ public void testFifoScheduling() throws Exception { Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses() .size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals( + 5 * GB, report_nm1.getUsedGuaranteedResource().getMemorySize()); rm.stop(); } @@ -822,7 +829,8 @@ private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) int checkAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals( + checkAlloc, report_nm1.getUsedGuaranteedResource().getMemorySize()); rm.stop(); } @@ -1113,8 +1121,10 @@ public void testResourceOverCommit() throws Exception { SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 2 GB used and 2 GB available - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); @@ -1135,8 +1145,10 @@ public void testResourceOverCommit() throws Exception { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); @@ -1155,7 +1167,7 @@ public void testResourceOverCommit() throws Exception { while (waitCount++ != 20) { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); if (null != report_nm1 && - report_nm1.getAvailableResource().getMemorySize() != 0) { + report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) { break; } LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried " @@ -1165,8 +1177,10 @@ public void testResourceOverCommit() throws Exception { // Now, the used resource is still 4 GB, and available resource is minus // value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); + Assert.assertEquals(-2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Check container can complete successfully in case of resource // over-commitment. @@ -1184,9 +1198,11 @@ public void testResourceOverCommit() throws Exception { Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses() .size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getUsedGuaranteedResource().getMemorySize()); // As container return 2 GB back, the available resource becomes 0 again. - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index fb597fc..d4ac154 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -811,15 +811,16 @@ public void verifyNodeInfoGeneric(RMNode node, String state, String rack, if (report != null) { assertEquals("numContainers doesn't match: " + numContainers, - report.getNumContainers(), numContainers); + report.getNumGuaranteedContainers(), numContainers); assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report - .getUsedResource().getMemorySize(), usedMemoryMB); + .getUsedGuaranteedResource().getMemorySize(), usedMemoryMB); assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report - .getAvailableResource().getMemorySize(), availMemoryMB); + .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB); assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report - .getUsedResource().getVirtualCores(), usedVirtualCores); - assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report - .getAvailableResource().getVirtualCores(), availVirtualCores); + .getUsedGuaranteedResource().getVirtualCores(), usedVirtualCores); + assertEquals("availVirtualCores doesn't match: " + availVirtualCores, + report.getAvailableGuaranteedResource().getVirtualCores(), + availVirtualCores); } if (opportunisticStatus != null) {