Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (revision 1562379) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (working copy) @@ -858,7 +858,7 @@ // Check queue max-capacity limit if (!assignToQueue(clusterResource, required)) { - return NULL_ASSIGNMENT; + break; } // Check user limit Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (revision 1562379) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (working copy) @@ -1227,6 +1227,103 @@ assignment.getExcessReservation().getContainer().getResource().getMemory()); } + @Test + public void testAllocationStarvationWithHighReservation() throws Exception { + // Manipulate queue 'a' + LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A)); + // unset maxCapacity + a.setMaxCapacity(1.0f); + + // Users + final String user_0 = "user_0"; + final String user_1 = "user_1"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = TestUtils + .getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, + mock(ActiveUsersManager.class), rmContext); + a.submitApplicationAttempt(app_0, user_0); + + final ApplicationAttemptId appAttemptId_1 = TestUtils + .getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a, + mock(ActiveUsersManager.class), rmContext); + a.submitApplicationAttempt(app_1, user_1); + + // Setup some nodes + String host_0 = "127.0.0.1"; + FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, + 8 * GB); + String host_1 = "127.0.0.2"; + FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, + 8 * GB); + + final int numNodes = 2; + Resource clusterResource = Resources.createResource(numNodes * (8 * GB), + numNodes * 16); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + + // Setup resource-requests considering as a AM request for App_0 + Priority priority_am = TestUtils.createMockPriority(1); + app_0.updateResourceRequests(Collections.singletonList(TestUtils + .createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, + priority_am, recordFactory))); + + // Setup resource-requests considering as a MAP request for App_0 + Priority priority_map = TestUtils.createMockPriority(10); + app_0.updateResourceRequests(Collections.singletonList(TestUtils + .createResourceRequest(ResourceRequest.ANY, 8 * GB, 2, true, + priority_map, recordFactory))); + + // Setup resource-requests considering as a AM request for App_1 + app_1.updateResourceRequests(Collections.singletonList(TestUtils + .createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, + priority_am, recordFactory))); + + // Start testing... + + // Only 1 container of 2Gb first + a.assignContainers(clusterResource, node_0); + assertEquals(2 * GB, a.getUsedResources().getMemory()); + assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(0 * GB, a.getMetrics().getReservedMB()); + assertEquals(2 * GB, a.getMetrics().getAllocatedMB()); + assertEquals(0 * GB, a.getMetrics().getAvailableMB()); + + // Now, reservation should come for App_0 for 8GB Map task + a.assignContainers(clusterResource, node_0); + assertEquals(10 * GB, a.getUsedResources().getMemory()); + assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(8 * GB, app_0.getCurrentReservation().getMemory()); + assertEquals(8 * GB, a.getMetrics().getReservedMB()); + + // Make sure proper re-reservation is made on same node to ensure + // starvation + a.assignContainers(clusterResource, node_0); + assertEquals(10 * GB, a.getUsedResources().getMemory()); + assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(8 * GB, app_0.getCurrentReservation().getMemory()); + assertEquals(8 * GB, a.getMetrics().getReservedMB()); + + a.assignContainers(clusterResource, node_0); + assertEquals(10 * GB, a.getUsedResources().getMemory()); + assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(8 * GB, app_0.getCurrentReservation().getMemory()); + assertEquals(8 * GB, a.getMetrics().getReservedMB()); + + // Now, try assign a container for App_1 in Node_1. This should be success + a.assignContainers(clusterResource, node_1); + assertEquals(12 * GB, a.getUsedResources().getMemory()); + assertEquals(2 * GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(8 * GB, a.getMetrics().getReservedMB()); + assertEquals(4 * GB, a.getMetrics().getAllocatedMB()); + assertEquals(0 * GB, a.getMetrics().getAvailableMB()); + } @Test public void testLocalityScheduling() throws Exception {