diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 475f2c7..3e423dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -596,7 +596,7 @@ public synchronized ApplicationResourceUsageReport getResourceUsageReport() { AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage(); Resource usedResourceClone = - Resources.clone(attemptResourceUsage.getUsed()); + Resources.clone(attemptResourceUsage.getAllUsed()); Resource reservedResourceClone = Resources.clone(attemptResourceUsage.getReserved()); return ApplicationResourceUsageReport.newInstance(liveContainers.size(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 598f279..ad2178a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -433,7 +433,7 @@ public synchronized User getUser(String userName) { for (Map.Entry entry : users.entrySet()) { User user = entry.getValue(); usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user - .getUsed()), user.getActiveApplications(), user + .getAllUsed()), user.getActiveApplications(), user .getPendingApplications(), Resources.clone(user .getConsumedAMResources()), Resources.clone(user .getUserResourceLimit()))); @@ -1888,7 +1888,11 @@ public ResourceUsage getResourceUsage() { public Resource getUsed() { return userResourceUsage.getUsed(); } - + + public Resource getAllUsed() { + return userResourceUsage.getAllUsed(); + } + public Resource getUsed(String label) { return userResourceUsage.getUsed(label); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index e60e496..c3c6d0a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -19,9 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.util.ArrayList; +import java.util.Iterator; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -34,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -95,6 +98,70 @@ private void checkUsedResource(MockRM rm, String queueName, int memory, .getMemory()); } + @Test(timeout = 60000) + public void testResourceUsage() throws Exception { + // set node -> label + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", + "z")); + + // set mapping: + // h1 -> x + // h2 -> y + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y"))); + + // inject node label manager + MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm.getRMContext().setNodeLabelManager(mgr); + rm.start(); + MockNM nm1 = rm.registerNode("h1:1234", 2048); + MockNM nm2 = rm.registerNode("h2:1234", 2048); + MockNM nm3 = rm.registerNode("h3:1234", 2048); + + ContainerId containerId; + + // launch an app to queue a1 (label = x), and check all container will + // be allocated in h1 + RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); + ApplicationResourceUsageReport appResourceUsageReport = + rm.getResourceScheduler().getAppResourceUsageReport( + am1.getApplicationAttemptId()); + Assert.assertEquals(1024, appResourceUsageReport.getUsedResources() + .getMemory()); + Assert.assertEquals(1, appResourceUsageReport.getUsedResources() + .getVirtualCores()); + // request a container. + am1.allocate("*", GB, 1, new ArrayList(), "x"); + containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); + rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000); + appResourceUsageReport = + rm.getResourceScheduler().getAppResourceUsageReport( + am1.getApplicationAttemptId()); + Assert.assertEquals(2048, appResourceUsageReport.getUsedResources() + .getMemory()); + Assert.assertEquals(2, appResourceUsageReport.getUsedResources() + .getVirtualCores()); + LeafQueue queue = + (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler()) + .getQueue("a"); + ArrayList users = queue.getUsers(); + for (UserInfo userInfo : users) { + if (userInfo.getUsername().equals("user")) { + ResourceInfo resourcesUsed = userInfo.getResourcesUsed(); + Assert.assertEquals(2048, resourcesUsed.getMemory()); + Assert.assertEquals(2, resourcesUsed.getvCores()); + } + } + rm.stop(); + } + @Test (timeout = 60000) public void testNodeUpdate() throws Exception { // set node -> label