Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (working copy) @@ -268,15 +268,20 @@ assertEquals("incorrect number of elements", 1, nodes.length()); JSONArray nodeArray = nodes.getJSONArray("node"); assertEquals("incorrect number of elements", 2, nodeArray.length()); - for (int i = 0; i < nodeArray.length(); ++i) { - JSONObject info = nodeArray.getJSONObject(i); - String host = info.get("id").toString().split(":")[0]; - RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(host); - WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", - info.getString("nodeHTTPAddress")); - WebServicesTestUtils.checkStringMatch("state", rmNode.getState() - .toString(), info.getString("state")); - } + + JSONObject info = nodeArray.getJSONObject(0); + RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nm1.getNodeId()); + WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", + info.getString("nodeHTTPAddress")); + WebServicesTestUtils.checkStringMatch("state", rmNode.getState() + .toString(), info.getString("state")); + + info = nodeArray.getJSONObject(1); + rmNode = rm.getRMContext().getInactiveRMNodes().get(nm2.getNodeId()); + WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", + info.getString("nodeHTTPAddress")); + WebServicesTestUtils.checkStringMatch("state", rmNode.getState() + .toString(), info.getString("state")); } @Test @@ -302,7 +307,7 @@ assertEquals("Incorrect Node Information.", "h2:1234", id); - RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get("h2"); + RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nm2.getNodeId()); WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", info.getString("nodeHTTPAddress")); WebServicesTestUtils.checkStringMatch("state", Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (working copy) @@ -157,10 +157,10 @@ final List deactivatedNodes = MockNodes.deactivatedNodes(racks, numNodes, newResource(mbsPerNode)); - final ConcurrentMap deactivatedNodesMap = + final ConcurrentMap deactivatedNodesMap = Maps.newConcurrentMap(); for (RMNode node : deactivatedNodes) { - deactivatedNodesMap.put(node.getHostName(), node); + deactivatedNodesMap.put(node.getNodeID(), node); } return new RMContextImpl(null, null, null, null, null, null, null, null, null) { @@ -169,7 +169,7 @@ return applicationsMaps; } @Override - public ConcurrentMap getInactiveRMNodes() { + public ConcurrentMap getInactiveRMNodes() { return deactivatedNodesMap; } @Override Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (working copy) @@ -506,6 +506,35 @@ assertEquals(0, scheduler.rootMetrics.getReservedMB()); } + public void testSchedulerRootQueueMetricsWhenKillWaitingJobs() throws InterruptedException { + + // Add a node + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Queue 1 requests full capacity of node + createSchedulingRequest(1024, "queue1", "user1", 1); + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent); + + // Now adding another job, which have to wait since there is no spare capacity + ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory()); + // Make sure reserved memory gets updated correctly + assertEquals(1024, scheduler.rootMetrics.getReservedMB()); + + // Now remove this waiting app, and the reserved memory should be released. + AppRemovedSchedulerEvent appRemovedEvent = new AppRemovedSchedulerEvent( + attId, RMAppAttemptState.KILLED); + scheduler.handle(appRemovedEvent); + assertEquals(0, scheduler.rootMetrics.getReservedMB()); + } + @Test (timeout = 5000) public void testSimpleContainerAllocation() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (working copy) @@ -453,11 +453,10 @@ new NodesListManagerEvent( NodesListManagerEventType.NODE_USABLE, rmNode)); - String host = rmNode.nodeId.getHost(); - if (rmNode.context.getInactiveRMNodes().containsKey(host)) { + RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(rmNode.nodeId); + if (previouRMNode != null) { // Old node rejoining - RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(host); - rmNode.context.getInactiveRMNodes().remove(host); + rmNode.context.getInactiveRMNodes().remove(rmNode.nodeId); rmNode.updateMetricsForRejoinedNode(previouRMNode.getState()); } else { // Increment activeNodes explicitly because this is a new node. @@ -554,7 +553,7 @@ rmNode.context.getRMNodes().remove(rmNode.nodeId); LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + finalState); - rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode); + rmNode.context.getInactiveRMNodes().put(rmNode.nodeId, rmNode); //Update the metrics rmNode.updateMetricsForDeactivatedNode(initialState, finalState); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (working copy) @@ -51,8 +51,8 @@ private final ConcurrentMap nodes = new ConcurrentHashMap(); - private final ConcurrentMap inactiveNodes - = new ConcurrentHashMap(); + private final ConcurrentMap inactiveNodes + = new ConcurrentHashMap(); private AMLivelinessMonitor amLivelinessMonitor; private AMLivelinessMonitor amFinishingMonitor; @@ -132,7 +132,7 @@ } @Override - public ConcurrentMap getInactiveRMNodes() { + public ConcurrentMap getInactiveRMNodes() { return this.inactiveNodes; } @@ -290,4 +290,4 @@ ResourceTrackerService resourceTrackerService) { this.resourceTrackerService = resourceTrackerService; } -} \ No newline at end of file +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (working copy) @@ -47,7 +47,7 @@ ConcurrentMap getRMApps(); - ConcurrentMap getInactiveRMNodes(); + ConcurrentMap getInactiveRMNodes(); ConcurrentMap getRMNodes(); @@ -85,4 +85,4 @@ void setRMDelegationTokenSecretManager( RMDelegationTokenSecretManager delegationTokenSecretManager); -} \ No newline at end of file +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (working copy) @@ -234,7 +234,7 @@ RMNode ni = this.rm.getRMContext().getRMNodes().get(nid); boolean isInactive = false; if (ni == null) { - ni = this.rm.getRMContext().getInactiveRMNodes().get(nid.getHost()); + ni = this.rm.getRMContext().getInactiveRMNodes().get(nid); if (ni == null) { throw new NotFoundException("nodeId, " + nodeId + ", is not found"); } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (revision 1549996) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (working copy) @@ -751,6 +751,8 @@ if (rmContainer.getState() == RMContainerState.RESERVED) { application.unreserve(node, rmContainer.getReservedPriority()); node.unreserveResource(application); + getRootQueueMetrics().unreserveResource( + application.getUser(), rmContainer.getContainer().getResource()); } else { application.containerCompleted(rmContainer, containerStatus, event); node.releaseContainer(container);