diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java index 2ae4872..a2f76cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -169,4 +169,18 @@ public void subtractFrom(int pmem, int vmem, float cpu) { this.setVirtualMemory(this.getVirtualMemory() - vmem); this.setCPU(this.getCPU() - cpu); } + + /** + * Convert utilization into a resource allocation. It ceils the values to fit + * into container sizes. + * @return New resource for this utilization. + */ + @Public + @Unstable + public Resource toResource() { + long mem = this.getVirtualMemory(); + long vcores = (long)Math.ceil(this.getCPU()); + Resource resource = Resource.newInstance(mem, vcores); + return resource; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 1f57e07..b237cb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -54,7 +54,6 @@ private static final Log LOG = LogFactory.getLog(SchedulerNode.class); - private Resource unallocatedResource = Resource.newInstance(0, 0); private Resource allocatedResource = Resource.newInstance(0, 0); private Resource totalResource; private RMContainer reservedContainer; @@ -76,7 +75,6 @@ public SchedulerNode(RMNode node, boolean usePortForNodeName, Set labels) { this.rmNode = node; - this.unallocatedResource = Resources.clone(node.getTotalCapability()); this.totalResource = Resources.clone(node.getTotalCapability()); if (usePortForNodeName) { nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); @@ -100,8 +98,6 @@ public RMNode getRMNode() { */ public synchronized void updateTotalResource(Resource resource){ this.totalResource = resource; - this.unallocatedResource = Resources.subtract(totalResource, - this.allocatedResource); } /** @@ -211,7 +207,43 @@ public synchronized void decreaseContainer(ContainerId containerId, * @return Unallocated resources on the node */ public synchronized Resource getUnallocatedResource() { - return this.unallocatedResource; + Resource unallocatedResource = Resources.subtract( + this.getTotalResource(), this.getAllocatedResource()); + + // TODO add option to do this? + boolean scheduleExternalUtilization = true; + + if (scheduleExternalUtilization) { + ResourceUtilization externalUtilization = getExternalUtilization(); + if (externalUtilization != null) { + Resource externalResource = externalUtilization.toResource(); + Resources.subtractFrom(unallocatedResource, externalResource); + } + } + + return unallocatedResource; + } + + /** + * Estimate the external utilization in the node. + * @return Resource utilizaiton in the node or null if not available. + */ + private ResourceUtilization getExternalUtilization() { + ResourceUtilization externalUtilization = null; + + ResourceUtilization nodeUtilization = rmNode.getNodeUtilization(); + ResourceUtilization containersUtilization = + rmNode.getAggregatedContainersUtilization(); + + if (nodeUtilization != null && containersUtilization != null) { + externalUtilization = ResourceUtilization.newInstance(nodeUtilization); + externalUtilization.subtractFrom( + containersUtilization.getPhysicalMemory(), + containersUtilization.getVirtualMemory(), + containersUtilization.getCPU()); + } + + return externalUtilization; } /** @@ -286,7 +318,6 @@ private synchronized void addUnallocatedResource(Resource resource) { + rmNode.getNodeAddress()); return; } - Resources.addTo(unallocatedResource, resource); Resources.subtractFrom(allocatedResource, resource); } @@ -301,7 +332,6 @@ private synchronized void deductUnallocatedResource(Resource resource) { + rmNode.getNodeAddress()); return; } - Resources.subtractFrom(unallocatedResource, resource); Resources.addTo(allocatedResource, resource); }