diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index d35ef96..46f4958 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -700,7 +700,7 @@ public synchronized Resource allowedResourceForOverAllocation() { } ResourceUtilization projectedNodeUtilization = ResourceUtilization. - newInstance(getNodeUtilization()); + newInstance(getAggregatedContainersUtilization()); // account for resources allocated in this heartbeat projectedNodeUtilization.addTo( (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 8d3a495..00c4bc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -313,18 +313,14 @@ public Integer getDecommissioningTimeout() { public Resource getPhysicalResource() { return this.physicalResource; } - - public void updateResourceUtilization(ResourceUtilization utilization) { - this.nodeUtilization = utilization; - } - - public void updateContainersAndNodeUtilization( + + public void updateContainersInfoAndUtilization( UpdatedContainerInfo updatedContainerInfo, ResourceUtilization resourceUtilization) { if (updatedContainerInfo != null) { containerUpdates = Collections.singletonList(updatedContainerInfo); } - this.nodeUtilization = resourceUtilization; + this.containersUtilization = resourceUtilization; } }; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 7fbf84a..3513838 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -2758,7 +2758,7 @@ public void testResourceRequestOptOutOfOversubscription() throws Exception { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -2850,7 +2850,7 @@ public void testQueueOptOutOfOversubscription() throws IOException { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -2942,7 +2942,7 @@ public void testAllocateNoOpportunisticContainersOnBusyNode() ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(2000, 0, 0.8f)); @@ -3026,7 +3026,7 @@ public void testAllocateOpportunisticContainersOnPartiallyOverAllocatedNode() ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1800, 0, 0.5f)); @@ -3112,7 +3112,7 @@ public void testAllocateOpportunisticContainersOnFullyAllocatedNode() ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1800, 0, 0.5f)); @@ -3199,7 +3199,7 @@ public void testAllocateOpportunisticContainersWithGuaranteedOnes() ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(512, 0, 0.1f)); @@ -3300,7 +3300,7 @@ public void testMaxOverallocationPerNode() throws Exception { ContainerStatus containerStatus1 = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus1), Collections.emptyList()), ResourceUtilization.newInstance(0, 0, 0.0f)); @@ -3323,7 +3323,7 @@ public void testMaxOverallocationPerNode() throws Exception { ContainerStatus containerStatus2 = ContainerStatus.newInstance( allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus2), Collections.emptyList()), ResourceUtilization.newInstance(0, 0, 0.0f)); @@ -3418,7 +3418,7 @@ public void testSingleOpportunisticContainerPromotionWithoutReservation() List containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3442,7 +3442,7 @@ public void testSingleOpportunisticContainerPromotionWithoutReservation() List finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3527,7 +3527,7 @@ public void testMultipleOpportunisticContainerPromotionWithoutReservation() List containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3553,7 +3553,7 @@ public void testMultipleOpportunisticContainerPromotionWithoutReservation() ContainerStatus container3Status = ContainerStatus.newInstance( allocatedContainers3.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(container3Status), Collections.emptyList()), ResourceUtilization.newInstance(2000, 0, 0.2f)); @@ -3578,7 +3578,7 @@ public void testMultipleOpportunisticContainerPromotionWithoutReservation() List finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3597,7 +3597,7 @@ public void testMultipleOpportunisticContainerPromotionWithoutReservation() finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3684,7 +3684,7 @@ public void testOpportunisticContainerPromotionWithPriorReservation() List containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3723,7 +3723,7 @@ public void testOpportunisticContainerPromotionWithPriorReservation() List finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3749,7 +3749,7 @@ public void testOpportunisticContainerPromotionWithPriorReservation() finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3838,7 +3838,7 @@ public void testOpportunisticContainerPromotionWithPostReservation() List containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3876,7 +3876,7 @@ public void testOpportunisticContainerPromotionWithPostReservation() List finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3894,7 +3894,7 @@ public void testOpportunisticContainerPromotionWithPostReservation() finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node));