diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultHeadroomProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultHeadroomProvider.java new file mode 100644 index 0000000..c094f0f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultHeadroomProvider.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.yarn.api.records.Resource; + +public class DefaultHeadroomProvider implements HeadroomProvider { + + private Resource headroom = Resource.newInstance(0, 0); + + @Override + public void setHeadroom(Resource headroom) { + this.headroom = headroom; + } + + @Override + public Resource getHeadroom() { + // Corner case to deal with applications being slightly over-limit + if (headroom.getMemory() < 0) { + headroom.setMemory(0); + } + return headroom; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/HeadroomProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/HeadroomProvider.java new file mode 100644 index 0000000..23c8ceb --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/HeadroomProvider.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Implementers accept and provide headroom information for applications, + * potentially recalculating on demand to provide more up-to-date information + */ +public interface HeadroomProvider { + + public void setHeadroom(Resource headroom); + + public Resource getHeadroom(); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 933f456..7534cda 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -79,12 +79,13 @@ private final Multiset reReservations = HashMultiset.create(); protected final Resource currentReservation = Resource.newInstance(0, 0); - private Resource resourceLimit = Resource.newInstance(0, 0); protected Resource currentConsumption = Resource.newInstance(0, 0); private Resource amResource; private boolean unmanagedAM = true; private boolean amRunning = false; + private HeadroomProvider headroomProvider = new DefaultHeadroomProvider(); + protected List newlyAllocatedContainers = new ArrayList(); @@ -215,6 +216,14 @@ public void setAmRunning(boolean bool) { amRunning = bool; } + public synchronized void setHeadroomProvider(HeadroomProvider headroomProvider) { + this.headroomProvider = headroomProvider; + } + + public synchronized HeadroomProvider getHeadroomProvider() { + return headroomProvider; + } + public boolean getUnmanagedAM() { return unmanagedAM; } @@ -342,7 +351,7 @@ public synchronized boolean isReserved(SchedulerNode node, Priority priority) { } public synchronized void setHeadroom(Resource globalLimit) { - this.resourceLimit = globalLimit; + headroomProvider.setHeadroom(globalLimit); } /** @@ -350,12 +359,7 @@ public synchronized void setHeadroom(Resource globalLimit) { * @return available resource headroom */ public synchronized Resource getHeadroom() { - // Corner case to deal with applications being slightly over-limit - if (resourceLimit.getMemory() < 0) { - resourceLimit.setMemory(0); - } - - return resourceLimit; + return headroomProvider.getHeadroom(); } public synchronized int getNumReservedContainers(Priority priority) { @@ -518,7 +522,7 @@ public synchronized ApplicationResourceUsageReport getResourceUsageReport() { } public synchronized Resource getResourceLimit() { - return this.resourceLimit; + return headroomProvider.getHeadroom(); } public synchronized Map getLastScheduledContainer() { @@ -530,7 +534,7 @@ public synchronized void transferStateFromPreviousAttempt( this.liveContainers = appAttempt.getLiveContainersMap(); // this.reReservations = appAttempt.reReservations; this.currentConsumption = appAttempt.getCurrentConsumption(); - this.resourceLimit = appAttempt.getResourceLimit(); + this.headroomProvider = appAttempt.getHeadroomProvider(); // this.currentReservation = appAttempt.currentReservation; // this.newlyAllocatedContainers = appAttempt.newlyAllocatedContainers; // this.schedulingOpportunities = appAttempt.schedulingOpportunities; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java new file mode 100644 index 0000000..f713ab0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultHeadroomProvider; + +public class CapacityHeadroomProvider extends DefaultHeadroomProvider { + + LeafQueue.User user; + LeafQueue queue; + FiCaSchedulerApp application; + Resource required; + LeafQueue.QueueHeadroomInfo queueHeadroomInfo; + + public CapacityHeadroomProvider( + LeafQueue.User user, + LeafQueue queue, + FiCaSchedulerApp application, + Resource required, + LeafQueue.QueueHeadroomInfo queueHeadroomInfo) { + + this.user = user; + this.queue = queue; + this.application = application; + this.required = required; + this.queueHeadroomInfo = queueHeadroomInfo; + + } + + @Override + public Resource getHeadroom() { + + Resource queueMaxCap; + Resource clusterResource; + synchronized (queueHeadroomInfo) { + queueMaxCap = queueHeadroomInfo.getQueueMaxCap(); + clusterResource = queueHeadroomInfo.getClusterResource(); + } + Resource headroom = queue.getHeadroom(user, queueMaxCap, clusterResource, application, required); + + // Corner case to deal with applications being slightly over-limit + if (headroom.getMemory() < 0) { + headroom.setMemory(0); + } + return headroom; + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c8a73bf..097392e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -464,6 +464,10 @@ private void reinitializeQueues(CapacitySchedulerConfiguration conf) // Re-configure queues root.reinitialize(newRoot, clusterResource); initializeQueueMappings(); + + // Re-calculate headroom for active applications + root.updateClusterResource(clusterResource); + } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 5c93c5f..f64a6ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -129,6 +129,8 @@ private final ResourceCalculator resourceCalculator; + private final QueueHeadroomInfo queueHeadroomInfo = new QueueHeadroomInfo(); + public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) { this.scheduler = cs; @@ -965,6 +967,22 @@ private synchronized boolean assignToQueue(Resource clusterResource, } return true; } + + protected Resource getHeadroom(User user, Resource queueMaxCap, + Resource clusterResource, FiCaSchedulerApp application, Resource required) { + return getHeadroom(user, queueMaxCap, clusterResource, + computeUserLimit(application, clusterResource, required, user)); + } + + private Resource getHeadroom(User user, Resource queueMaxCap, + Resource clusterResource, Resource userLimit) { + Resource headroom = + Resources.subtract( + Resources.min(resourceCalculator, clusterResource, + userLimit, queueMaxCap), + user.getConsumedResources()); + return headroom; + } @Lock({LeafQueue.class, FiCaSchedulerApp.class}) private Resource computeUserLimitAndSetHeadroom( @@ -972,12 +990,14 @@ private Resource computeUserLimitAndSetHeadroom( String user = application.getUser(); + User queueUser = getUser(user); + /** * Headroom is min((userLimit, queue-max-cap) - consumed) */ Resource userLimit = // User limit - computeUserLimit(application, clusterResource, required); + computeUserLimit(application, clusterResource, required, queueUser); //Max avail capacity needs to take into account usage by ancestor-siblings //which are greater than their base capacity, so we are interested in "max avail" @@ -991,23 +1011,28 @@ private Resource computeUserLimitAndSetHeadroom( clusterResource, absoluteMaxAvailCapacity, minimumAllocation); + + synchronized (queueHeadroomInfo) { + queueHeadroomInfo.setQueueMaxCap(queueMaxCap); + queueHeadroomInfo.setClusterResource(clusterResource); + } - Resource userConsumed = getUser(user).getConsumedResources(); - Resource headroom = - Resources.subtract( - Resources.min(resourceCalculator, clusterResource, - userLimit, queueMaxCap), - userConsumed); + Resource headroom = getHeadroom(queueUser, queueMaxCap, clusterResource, userLimit); if (LOG.isDebugEnabled()) { LOG.debug("Headroom calculation for user " + user + ": " + " userLimit=" + userLimit + " queueMaxCap=" + queueMaxCap + - " consumed=" + userConsumed + + " consumed=" + queueUser.getConsumedResources() + " headroom=" + headroom); } - application.setHeadroom(headroom); + CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider( + queueUser, this, application, required, queueHeadroomInfo + ); + + application.setHeadroomProvider(headroomProvider); + metrics.setAvailableResourcesToUser(user, headroom); return userLimit; @@ -1015,7 +1040,7 @@ private Resource computeUserLimitAndSetHeadroom( @Lock(NoLock.class) private Resource computeUserLimit(FiCaSchedulerApp application, - Resource clusterResource, Resource required) { + Resource clusterResource, Resource required, User user) { // What is our current capacity? // * It is equal to the max(required, queue-capacity) if // we're running below capacity. The 'max' ensures that jobs in queues @@ -1072,7 +1097,7 @@ private Resource computeUserLimit(FiCaSchedulerApp application, " userLimit=" + userLimit + " userLimitFactor=" + userLimitFactor + " required: " + required + - " consumed: " + getUser(userName).getConsumedResources() + + " consumed: " + user.getConsumedResources() + " limit: " + limit + " queueCapacity: " + queueCapacity + " qconsumed: " + usedResources + @@ -1466,7 +1491,6 @@ synchronized void allocateResource(Resource clusterResource, String userName = application.getUser(); User user = getUser(userName); user.assignContainer(resource); - Resources.subtractFrom(application.getHeadroom(), resource); // headroom metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { @@ -1661,4 +1685,29 @@ public void detachContainer(Resource clusterResource, getParent().detachContainer(clusterResource, application, rmContainer); } } + + /* + * Holds shared values used by all applications in + * the queue to calculate headroom on demand + */ + static class QueueHeadroomInfo { + private Resource queueMaxCap; + private Resource clusterResource; + + public void setQueueMaxCap(Resource queueMaxCap) { + this.queueMaxCap = queueMaxCap; + } + + public Resource getQueueMaxCap() { + return queueMaxCap; + } + + public void setClusterResource(Resource clusterResource) { + this.clusterResource = clusterResource; + } + + public Resource getClusterResource() { + return clusterResource; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index a9a9975..2083b3f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -518,7 +518,7 @@ public void testHeadroom() throws Exception { // Schedule to compute queue.assignContainers(clusterResource, node_0); Resource expectedHeadroom = Resources.createResource(10*16*GB, 1); - verify(app_0_0).setHeadroom(eq(expectedHeadroom)); + assertEquals(expectedHeadroom, app_0_0.getHeadroom()); // Submit second application from user_0, check headroom final ApplicationAttemptId appAttemptId_0_1 = @@ -536,8 +536,8 @@ public void testHeadroom() throws Exception { // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute - verify(app_0_0, times(2)).setHeadroom(eq(expectedHeadroom)); - verify(app_0_1).setHeadroom(eq(expectedHeadroom));// no change + assertEquals(expectedHeadroom, app_0_0.getHeadroom()); + assertEquals(expectedHeadroom, app_0_1.getHeadroom());// no change // Submit first application from user_1, check for new headroom final ApplicationAttemptId appAttemptId_1_0 = @@ -556,17 +556,17 @@ public void testHeadroom() throws Exception { // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute expectedHeadroom = Resources.createResource(10*16*GB / 2, 1); // changes - verify(app_0_0).setHeadroom(eq(expectedHeadroom)); - verify(app_0_1).setHeadroom(eq(expectedHeadroom)); - verify(app_1_0).setHeadroom(eq(expectedHeadroom)); + assertEquals(expectedHeadroom, app_0_0.getHeadroom()); + assertEquals(expectedHeadroom, app_0_1.getHeadroom()); + assertEquals(expectedHeadroom, app_1_0.getHeadroom()); // Now reduce cluster size and check for the smaller headroom clusterResource = Resources.createResource(90*16*GB); queue.assignContainers(clusterResource, node_0); // Schedule to compute expectedHeadroom = Resources.createResource(9*16*GB / 2, 1); // changes - verify(app_0_0).setHeadroom(eq(expectedHeadroom)); - verify(app_0_1).setHeadroom(eq(expectedHeadroom)); - verify(app_1_0).setHeadroom(eq(expectedHeadroom)); + assertEquals(expectedHeadroom, app_0_0.getHeadroom()); + assertEquals(expectedHeadroom, app_0_1.getHeadroom()); + assertEquals(expectedHeadroom, app_1_0.getHeadroom()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index d5eb933..173960f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -624,6 +624,91 @@ public void testUserLimits() throws Exception { } @Test + public void testUserHeadroomMultiApp() throws Exception { + // Mock the queue + LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + //unset maxCapacity + a.setMaxCapacity(1.0f); + + // Users + final String user_0 = "user_0"; + final String user_1 = "user_1"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_0, a, + a.getActiveUsersManager(), rmContext); + a.submitApplicationAttempt(app_0, user_0); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_0, a, + a.getActiveUsersManager(), rmContext); + a.submitApplicationAttempt(app_1, user_0); // same user + + final ApplicationAttemptId appAttemptId_2 = + TestUtils.getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = + new FiCaSchedulerApp(appAttemptId_2, user_1, a, + a.getActiveUsersManager(), rmContext); + a.submitApplicationAttempt(app_2, user_1); + + // Setup some nodes + String host_0 = "127.0.0.1"; + FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 16*GB); + String host_1 = "127.0.0.2"; + FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 16*GB); + + final int numNodes = 2; + Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + + Priority priority = TestUtils.createMockPriority(1); + + app_0.updateResourceRequests(Collections.singletonList( + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, + priority, recordFactory))); + + a.assignContainers(clusterResource, node_0); + assertEquals(1*GB, a.getUsedResources().getMemory()); + assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); + //Now, headroom is the same for all apps for a given user + queue combo + //and a change to any app's headroom is reflected for all the user's apps + //once those apps are active/have themselves calculated headroom for allocation + //at least one time + assertEquals(2*GB, app_0.getHeadroom().getMemory()); + assertEquals(0*GB, app_1.getHeadroom().getMemory());//not yet active + assertEquals(0*GB, app_2.getHeadroom().getMemory());//not yet active + + app_1.updateResourceRequests(Collections.singletonList( + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, + priority, recordFactory))); + + a.assignContainers(clusterResource, node_0); + assertEquals(2*GB, a.getUsedResources().getMemory()); + assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); + assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); + assertEquals(1*GB, app_0.getHeadroom().getMemory()); + assertEquals(1*GB, app_1.getHeadroom().getMemory());//now active + assertEquals(0*GB, app_2.getHeadroom().getMemory());//not yet active + + //Complete container and verify that headroom is updated, for both apps for the user + RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); + + assertEquals(2*GB, app_0.getHeadroom().getMemory()); + assertEquals(2*GB, app_1.getHeadroom().getMemory()); + } + + @Test public void testHeadroomWithMaxCap() throws Exception { // Mock the queue LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); @@ -694,16 +779,16 @@ public void testHeadroomWithMaxCap() throws Exception { assertEquals(2*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G - assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G + assertEquals(2*GB, app_0.getHeadroom().getMemory()); // User limit = 4G, 2 in use + assertEquals(0*GB, app_1.getHeadroom().getMemory()); // the application is not yet active // Again one to user_0 since he hasn't exceeded user limit yet a.assignContainers(clusterResource, node_0); assertEquals(3*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G - assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G + assertEquals(1*GB, app_0.getHeadroom().getMemory()); // 4G - 3G + assertEquals(1*GB, app_1.getHeadroom().getMemory()); // 4G - 3G // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f);