diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 825c398..9c79841 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -171,6 +171,34 @@ private synchronized void unreserveInternal( + priority + "; currentReservation " + currentReservation); } + @Override + public synchronized Resource getHeadroom() { + final FSQueue queue = (FSQueue) this.queue; + SchedulingPolicy policy = queue.getPolicy(); + Resource queueFairShare = queue.getFairShare(); + Resource queueUsage = queue.getResourceUsage(); + Resource queueFairShareAvailable = policy.getAvailable(queueFairShare, + queueUsage); + Resource clusterResource = this.scheduler.getClusterResource(); + Resource clusterUsage = this.scheduler.getQueueManager() + .getRootQueue().getResourceUsage(); + Resource clusterAvailableResource = Resources.subtract(clusterResource, + clusterUsage); + Resource headroom = policy.getEffectiveAvailable( + clusterAvailableResource, queueFairShareAvailable); + if (LOG.isDebugEnabled()) { + LOG.debug("Headroom calculation for " + this.getName() + ":" + + "queueFairShareAvailable=" + queueFairShareAvailable + + "(queueFairShare=" + queueFairShare + + " - queueUsage=" + queueUsage + ") " + + " clusterAvailableResource=" + clusterAvailableResource + + "(clusterResource=" + clusterResource + + " - clusterUsage=" + clusterUsage + ")" + + "Headroom=" + headroom); + } + return headroom; + } + public synchronized float getLocalityWaitFactor( Priority priority, int clusterNodes) { // Estimate: Required unique resources (i.e. hosts + racks) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index ca006c5..cba91c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -175,4 +175,29 @@ public abstract boolean checkIfUsageOverFairShare( */ public abstract boolean checkIfAMResourceUsageOverLimit( Resource usage, Resource maxAMResource); + + /** + * Subtract used resources that are applicable to this + * policy from max to get available resource. For eg if only + * memory then leave other resources such as CPU to same as maxAvailable. + * + * @param maxAvailable maximum available resource + * @param used used resources + * @return resultant resource + */ + public abstract Resource getAvailable(Resource maxAvailable, Resource used); + + /** + * Get minimum from clusterAvailable and + * fairShareAvailable, resources that are applicable to this + * policy. For eg if only memory then leave other resources such as CPU to + * same as clusterAvailable. + * + * @param clusterAvailable available resource in cluster + * @param fairShareAvailable fairshare of resources + * @return resultant resource + */ + public abstract Resource getEffectiveAvailable(Resource clusterAvailable, + Resource fairShareAvailable); + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 42044bc..fa4dae6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -77,7 +77,23 @@ public void computeSteadyShares(Collection queues, ComputeFairShares.computeSteadyShares(queues, totalResources, type); } } - + + @Override + public Resource getAvailable(Resource maxAvailable, Resource used) { + return Resources.createResource(Math.max(maxAvailable.getMemory() - + used.getMemory(), 0), Math.max(maxAvailable.getVirtualCores() - + used.getVirtualCores(), 0)); + } + + @Override + public Resource getEffectiveAvailable(Resource clusterAvailable, + Resource fairShareAvailable) { + return Resources.createResource(Math.min(clusterAvailable.getMemory(), + fairShareAvailable.getMemory()), Math.min(clusterAvailable + .getVirtualCores(), + fairShareAvailable.getVirtualCores())); + } + @Override public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) { return !Resources.fitsIn(usage, fairShare); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 66bb88b..03a3ecd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -115,6 +115,19 @@ else if (s1Needy && s2Needy) } @Override + public Resource getAvailable(Resource maxAvailable, Resource used) { + return Resources.createResource(Math.max(maxAvailable.getMemory() - + used.getMemory(), 0), maxAvailable.getVirtualCores()); + } + + @Override + public Resource getEffectiveAvailable(Resource clusterAvailable, + Resource fairShareAvailable) { + return Resources.createResource(Math.min(clusterAvailable.getMemory(), + fairShareAvailable.getMemory()), clusterAvailable.getVirtualCores()); + } + + @Override public void computeShares(Collection schedulables, Resource totalResources) { ComputeFairShares.computeShares(schedulables, totalResources, ResourceType.MEMORY); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index 591ee49..336f149 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -108,6 +108,19 @@ public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMRes } @Override + public Resource getAvailable(Resource maxAvailable, Resource used) { + return Resources.createResource(Math.max(maxAvailable.getMemory() - + used.getMemory(), 0), maxAvailable.getVirtualCores()); + } + + @Override + public Resource getEffectiveAvailable(Resource clusterAvailable, + Resource fairShareAvailable) { + return Resources.createResource(Math.min(clusterAvailable.getMemory(), + fairShareAvailable.getMemory()), clusterAvailable.getVirtualCores()); + } + + @Override public byte getApplicableDepth() { return SchedulingPolicy.DEPTH_LEAF; }