diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index f0d1ed1..087766a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -182,18 +182,26 @@ public Resource getHeadroom() { Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterUsage = this.scheduler.getRootQueueMetrics() .getAllocatedResources(); - Resource clusterAvailableResource = Resources.subtract(clusterResource, - clusterUsage); + + // Max available resources for this queue correspond to the minimum + // of resources available on the cluster and queue's maxResources + Resource maxAvailableResource = + Resources.subtract(clusterResource, clusterUsage); + maxAvailableResource.setMemory( + Math.min(maxAvailableResource.getMemory(), + queue.getMaxShare().getMemory())); + maxAvailableResource.setVirtualCores( + Math.min(maxAvailableResource.getVirtualCores(), + queue.getMaxShare().getVirtualCores())); + Resource headroom = policy.getHeadroom(queueFairShare, - queueUsage, clusterAvailableResource); + queueUsage, maxAvailableResource); if (LOG.isDebugEnabled()) { LOG.debug("Headroom calculation for " + this.getName() + ":" + "Min(" + "(queueFairShare=" + queueFairShare + " - queueUsage=" + queueUsage + ")," + - " clusterAvailableResource=" + clusterAvailableResource + - "(clusterResource=" + clusterResource + - " - clusterUsage=" + clusterUsage + ")" + + " maxAvailableResource=" + maxAvailableResource + "Headroom=" + headroom); } return headroom; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index bf2a25b..abdc834 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -185,10 +185,10 @@ public abstract boolean checkIfAMResourceUsageOverLimit( * * @param queueFairShare fairshare in the queue * @param queueUsage resources used in the queue - * @param clusterAvailable available resource in cluster + * @param maxAvailable available resource in cluster for this queue * @return calculated headroom */ public abstract Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable); + Resource queueUsage, Resource maxAvailable); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 3f6cbd1..86d503b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -90,15 +90,15 @@ public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMRes @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, - Resource clusterAvailable) { + Resource maxAvailable) { int queueAvailableMemory = Math.max(queueFairShare.getMemory() - queueUsage.getMemory(), 0); int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage .getVirtualCores(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - Math.min(clusterAvailable.getVirtualCores(), + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + Math.min(maxAvailable.getVirtualCores(), queueAvailableCPU)); return headroom; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 97669cb..918db9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -116,12 +116,12 @@ else if (s1Needy && s2Needy) @Override public Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable) { + Resource queueUsage, Resource maxAvailable) { int queueAvailableMemory = Math.max( queueFairShare.getMemory() - queueUsage.getMemory(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - clusterAvailable.getVirtualCores()); + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + maxAvailable.getVirtualCores()); return headroom; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index a2e17ec..7d88933 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -109,12 +109,12 @@ public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMRes @Override public Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable) { + Resource queueUsage, Resource maxAvailable) { int queueAvailableMemory = Math.max( queueFairShare.getMemory() - queueUsage.getMemory(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - clusterAvailable.getVirtualCores()); + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + maxAvailable.getVirtualCores()); return headroom; }