diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 6f8144d..b5b22b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -116,14 +116,11 @@ public int getNewContainerId() { * The ApplicationMaster is updating resource requirements for the * application, by asking for more resources and releasing resources acquired * by the application. - * + * * @param requests resources to be acquired - * @param blacklistAdditions resources to be added to the blacklist - * @param blacklistRemovals resources to be removed from the blacklist */ synchronized public void updateResourceRequests( - List requests, - List blacklistAdditions, List blacklistRemovals) { + List requests) { QueueMetrics metrics = queue.getMetrics(); // Update resource requests @@ -181,11 +178,16 @@ synchronized public void updateResourceRequests( lastRequestContainers))); } } + } - // - // Update blacklist - // - + /** + * The ApplicationMaster is updating the blacklist + * + * @param blacklistAdditions resources to be added to the blacklist + * @param blacklistRemovals resources to be removed from the blacklist + */ + synchronized public void updateBlacklist( + List blacklistAdditions, List blacklistRemovals) { // Add to blacklist if (blacklistAdditions != null) { blacklist.addAll(blacklistAdditions); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2efb9ad..46a2dd6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -568,8 +568,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, application.showRequests(); // Update application requests - application.updateResourceRequests(ask, - blacklistAdditions, blacklistRemovals); + application.updateResourceRequests(ask); LOG.debug("allocate: post-update"); application.showRequests(); @@ -581,6 +580,8 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, " #ask=" + ask.size()); } + application.updateBlacklist(blacklistAdditions, blacklistRemovals); + return application.getAllocation(getResourceCalculator(), clusterResource, getMinimumResourceCapability()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index a261dbf..b93965c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -141,10 +141,16 @@ public String getUser() { } public synchronized void updateResourceRequests( - List requests, + List requests) { + if (!isStopped) { + this.appSchedulingInfo.updateResourceRequests(requests); + } + } + + public synchronized void updateBlacklist( List blacklistAdditions, List blacklistRemovals) { if (!isStopped) { - this.appSchedulingInfo.updateResourceRequests(requests, + this.appSchedulingInfo.updateBlacklist( blacklistAdditions, blacklistRemovals); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 670e961..0776937 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -138,7 +138,7 @@ public String getUser() { public synchronized void updateResourceRequests( List requests) { - this.appSchedulingInfo.updateResourceRequests(requests, null, null); + this.appSchedulingInfo.updateResourceRequests(requests); } public Map getResourceRequests(Priority priority) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 115d208..09641e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -295,7 +295,7 @@ public Allocation allocate( application.showRequests(); // Update application requests - application.updateResourceRequests(ask, blacklistAdditions, blacklistRemovals); + application.updateResourceRequests(ask); LOG.debug("allocate: post-update" + " applicationId=" + applicationAttemptId + @@ -307,6 +307,8 @@ public Allocation allocate( " #ask=" + ask.size()); } + application.updateBlacklist(blacklistAdditions, blacklistRemovals); + return new Allocation( application.pullNewlyAllocatedContainers(), application.getHeadroom()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f231685..f343bd5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -513,7 +513,7 @@ public void testHeadroom() throws Exception { app_0_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_0.updateResourceRequests(app_0_0_requests, null, null); + app_0_0.updateResourceRequests(app_0_0_requests); // Schedule to compute queue.assignContainers(clusterResource, node_0); @@ -532,7 +532,7 @@ public void testHeadroom() throws Exception { app_0_1_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_1.updateResourceRequests(app_0_1_requests, null, null); + app_0_1.updateResourceRequests(app_0_1_requests); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute @@ -551,7 +551,7 @@ public void testHeadroom() throws Exception { app_1_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_1_0.updateResourceRequests(app_1_0_requests, null, null); + app_1_0.updateResourceRequests(app_1_0_requests); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 40b73dc..f8b7ba5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -293,7 +293,7 @@ public void testSingleQueueOneUserMetrics() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Start testing... @@ -415,11 +415,11 @@ public void testSingleQueueWithOneUser() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Start testing... @@ -548,11 +548,11 @@ public void testUserLimits() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); /** * Start testing... @@ -573,7 +573,7 @@ public void testUserLimits() throws Exception { // Pre MAPREDUCE-3732 this test should fail without this block too // app_2.updateResourceRequests(Collections.singletonList( // TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority, -// recordFactory))); +// recordFactory))); // 1 container to user_0 a.assignContainers(clusterResource, node_0); @@ -641,11 +641,11 @@ public void testHeadroomWithMaxCap() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); /** * Start testing... @@ -680,7 +680,7 @@ public void testHeadroomWithMaxCap() throws Exception { a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); // No more to user_0 since he is already over user-limit @@ -697,7 +697,7 @@ public void testHeadroomWithMaxCap() throws Exception { LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true, - priority, recordFactory)), null, null); + priority, recordFactory))); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap @@ -758,11 +758,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory)), null, null); + priority, recordFactory))); /** * Start testing... @@ -792,11 +792,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_3.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Now allocations should goto app_2 since // user_0 is at limit inspite of high user-limit-factor @@ -920,11 +920,11 @@ public void testReservation() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Start testing... @@ -1024,7 +1024,7 @@ public void testStolenReservedContainer() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Setup app_1 to request a 4GB container on host_0 and // another 4GB container anywhere. @@ -1036,7 +1036,7 @@ public void testStolenReservedContainer() throws Exception { true, priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, true, priority, recordFactory)); - app_1.updateResourceRequests(appRequests_1, null, null); + app_1.updateResourceRequests(appRequests_1); // Start testing... @@ -1131,11 +1131,11 @@ public void testReservationExchange() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory)), null, null); + priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory)), null, null); + priority, recordFactory))); // Start testing... @@ -1260,7 +1260,7 @@ public void testLocalityScheduling() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); // Start testing... CSAssignment assignment = null; @@ -1325,7 +1325,7 @@ public void testLocalityScheduling() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); assertEquals(2, app_0.getTotalRequiredResources(priority)); String host_3 = "127.0.0.4"; // on rack_1 @@ -1416,7 +1416,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, priority_2, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); // Start testing... @@ -1531,7 +1531,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); // Start testing... @@ -1540,7 +1540,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); // NODE_LOCAL - node_0_1 a.assignContainers(clusterResource, node_0_0); @@ -1563,7 +1563,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); // No allocation on node_0_1 even though it's node/rack local since // required(rack_1) == 0 @@ -1808,8 +1808,8 @@ public void testLocalityConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, - Collections.singletonList(host_0_0), null); + app_0.updateResourceRequests(app_0_requests_0); + app_0.updateBlacklist(Collections.singletonList(host_0_0), null); app_0_requests_0.clear(); // @@ -1849,8 +1849,8 @@ public void testLocalityConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, - Collections.singletonList(host_1_1), null); + app_0.updateResourceRequests(app_0_requests_0); + app_0.updateBlacklist(Collections.singletonList(host_1_1), null); app_0_requests_0.clear(); // resourceName: @@ -1876,7 +1876,8 @@ public void testLocalityConstraints() throws Exception { assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 // Now, remove node_1_1 from blacklist, but add rack_1 to blacklist - app_0.updateResourceRequests(app_0_requests_0, + app_0.updateResourceRequests(app_0_requests_0); + app_0.updateBlacklist( Collections.singletonList(rack_1), Collections.singletonList(host_1_1)); app_0_requests_0.clear(); @@ -1903,8 +1904,8 @@ public void testLocalityConstraints() throws Exception { assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 // Now remove rack_1 from blacklist - app_0.updateResourceRequests(app_0_requests_0, - null, Collections.singletonList(rack_1)); + app_0.updateResourceRequests(app_0_requests_0); + app_0.updateBlacklist(null, Collections.singletonList(rack_1)); app_0_requests_0.clear(); // resourceName: @@ -1936,7 +1937,7 @@ public void testLocalityConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0.updateResourceRequests(app_0_requests_0); app_0_requests_0.clear(); // resourceName: