diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 41d3fd7..e9bf310 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -515,7 +515,7 @@ public synchronized void increaseContainer( } // Set queue metrics - queue.getMetrics().allocateResources(user, 0, + queue.getMetrics().allocateResources(user, increaseRequest.getDeltaCapacity(), true); // remove the increase request from pending increase request map @@ -539,7 +539,7 @@ public synchronized void decreaseContainer( } // Set queue metrics - queue.getMetrics().releaseResources(user, 0, absDelta); + queue.getMetrics().releaseResources(user, absDelta); // update usage appResourceUsage.decUsed(decreaseRequest.getNodePartition(), absDelta); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 68ae364..6c7778a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.resource.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -379,10 +378,9 @@ public void decrPendingResources(String user, int containers, Resource res) { } private void _decrPendingResources(int containers, Resource res) { - // if #container = 0, means change container resource pendingContainers.decr(containers); - pendingMB.decr(res.getMemory() * Math.max(containers, 1)); - pendingVCores.decr(res.getVirtualCores() * Math.max(containers, 1)); + pendingMB.decr(res.getMemory() * containers); + pendingVCores.decr(res.getVirtualCores() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -406,12 +404,11 @@ public void incrNodeTypeAggregations(String user, NodeType type) { public void allocateResources(String user, int containers, Resource res, boolean decrPending) { - // if #containers = 0, means change container resource allocatedContainers.incr(containers); aggregateContainersAllocated.incr(containers); - allocatedMB.incr(res.getMemory() * Math.max(containers, 1)); - allocatedVCores.incr(res.getVirtualCores() * Math.max(containers, 1)); + allocatedMB.incr(res.getMemory() * containers); + allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -424,12 +421,35 @@ public void allocateResources(String user, int containers, Resource res, } } + /** + * Allocate Resource for container size change + * + * @param user + * @param res + * @param decrPending + */ + public void allocateResources(String user, Resource res, + boolean decrPending) { + allocatedMB.incr(res.getMemory()); + allocatedVCores.incr(res.getVirtualCores()); + if (decrPending) { + pendingMB.decr(res.getMemory()); + pendingVCores.decr(res.getVirtualCores()); + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.allocateResources(user, res, decrPending); + } + if (parent != null) { + parent.allocateResources(user, res, decrPending); + } + } + public void releaseResources(String user, int containers, Resource res) { - // if #container = 0, means change container resource. allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); - allocatedMB.decr(res.getMemory() * Math.max(containers, 1)); - allocatedVCores.decr(res.getVirtualCores() * Math.max(containers, 1)); + allocatedMB.decr(res.getMemory() * containers); + allocatedVCores.decr(res.getVirtualCores() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -439,6 +459,24 @@ public void releaseResources(String user, int containers, Resource res) { } } + /** + * Release Resource for container size change + * + * @param user + * @param res + */ + public void releaseResources(String user, Resource res) { + allocatedMB.decr(res.getMemory()); + allocatedVCores.decr(res.getVirtualCores()); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.releaseResources(user, res); + } + if (parent != null) { + parent.releaseResources(user, res); + } + } + public void reserveResource(String user, Resource res) { reservedContainers.incr(); reservedMB.incr(res.getMemory()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 864620f..c772ae1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -87,6 +87,14 @@ public void setUp() { metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); + metrics.incrPendingResources(user, 0, Resources.createResource(2 * GB, 2)); + checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, + 0, 0, 0); + + metrics.decrPendingResources(user, 0, Resources.createResource(2 * GB, 2)); + checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, + 0, 0, 0); + metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);