diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 09fd73e..4c57525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -66,6 +66,7 @@ @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeInt availableMB; + @Metric("Max resource memory in MB") MutableGaugeInt queueMaxResource; @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB; @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; @@ -325,6 +326,14 @@ public void setAvailableResourcesToQueue(Resource limit) { } /** + * Set maximum resources to queue. + * @param limit resource limit + */ + public void setMaximumResourcesToQueue(Resource limit) { + queueMaxResource.set(limit.getMemory()); + } + + /** * Set available resources. To be called by scheduler periodically as * resources become available. * @param user @@ -510,6 +519,10 @@ public int getAllocatedContainers() { return allocatedContainers.value(); } + public int getQueueMaxResource(){ + return queueMaxResource.value(); + } + public int getAvailableMB() { return availableMB.value(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index f74106a..8c87ae3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -109,6 +109,13 @@ public void updateDemand() { demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); + Resource queueMaxRes = scheduler.getAllocationConfiguration() + .getMaxResources(childQueue.getName()); + if(Resources.equals(queueMaxRes,Resources.createResource(Integer.MAX_VALUE))) { + childQueue.getMetrics().setMaximumResourcesToQueue(maxRes); + } else { + childQueue.getMetrics().setMaximumResourcesToQueue(queueMaxRes); + } Resource toAdd = childQueue.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f481de5..c6281f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -307,6 +307,7 @@ protected synchronized void update() { rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); + rootQueue.getMetrics().setMaximumResourcesToQueue(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 8ad71d2..005f8f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -272,6 +272,32 @@ public void testQueueAppMetricsForMultipleFailures() { checkApps(parentUserSource, 1, 0, 0, 1, 0, 0, true); } + @Test public void testTwoLevelWithQueueMaxResourceMetrics() { + String parentQueueName = "root"; + String leafQueueName = "root.leaf"; + String leafQueueName1 = "root.leaf1"; + QueueMetrics parentMetrics = + QueueMetrics.forQueue(ms, parentQueueName, null, true, conf); + Queue parentQueue = make(stub(Queue.class).returning(parentMetrics). + from.getMetrics()); + QueueMetrics metrics = + QueueMetrics.forQueue(ms, leafQueueName, parentQueue, true, conf); + QueueMetrics metrics1 = + QueueMetrics.forQueue(ms, leafQueueName1, parentQueue, true, conf); + + MetricsSource parentQueueSource = queueSource(ms, parentQueueName); + MetricsSource queueSource = queueSource(ms, leafQueueName); + MetricsSource queueSource1 = queueSource(ms, leafQueueName1); + + parentMetrics.setMaximumResourcesToQueue(Resources.createResource(100*GB)); + metrics.setMaximumResourcesToQueue(Resources.createResource(50*GB)); + metrics1.setMaximumResourcesToQueue(Resources.createResource(50*GB)); + + checkQueueMaxResources(queueSource, 50*GB); + checkQueueMaxResources(queueSource1, 50*GB); + checkQueueMaxResources(parentQueueSource, 100*GB); + } + @Test public void testMetricsCache() { MetricsSystem ms = new MetricsSystemImpl("cache"); @@ -367,6 +393,11 @@ public static void checkResources(MetricsSource source, int allocatedMB, assertGauge("ReservedContainers", reservedCtnrs, rb); } + public static void checkQueueMaxResources(MetricsSource source, int queueMaxResource) { + MetricsRecordBuilder rb = getMetrics(source); + assertGauge("QueueMaxResource", queueMaxResource, rb); + } + private static AppSchedulingInfo mockApp(String user) { AppSchedulingInfo app = mock(AppSchedulingInfo.class); when(app.getUser()).thenReturn(user);