diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index e318d47..de6a310 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -136,7 +138,8 @@ synchronized public boolean updateResourceRequests( QueueMetrics metrics = queue.getMetrics(); boolean anyResourcesUpdated = false; - + HashMap anyPrioritymap = + new HashMap(); // Update resource requests for (ResourceRequest request : requests) { Priority priority = request.getPriority(); @@ -161,6 +164,11 @@ synchronized public boolean updateResourceRequests( if (request.getNumContainers() > 0) { activeUsersManager.activateApplication(user, applicationId); } + if ((null != request.getNodeLabelExpression()) + && (!request.getNodeLabelExpression().equals( + RMNodeLabelsManager.NO_LABEL))) { + anyPrioritymap.put(priority, request); + } } Map asks = this.requests.get(priority); @@ -214,6 +222,24 @@ synchronized public boolean updateResourceRequests( } } } + if (anyPrioritymap.size() > 0) { + for (ResourceRequest request : requests) { + String resourceName = request.getResourceName(); + if (resourceName.equals(ResourceRequest.ANY)) { + continue; + } + Priority priority = request.getPriority(); + ResourceRequest anyResourceRequest = anyPrioritymap.get(priority); + if ((null != anyResourceRequest)) { + request.setNodeLabelExpression(anyResourceRequest + .getNodeLabelExpression()); + if (LOG.isDebugEnabled()) { + LOG.debug("Updated label Expression for :" + " application= " + + applicationId + " request= " + request); + } + } + } + } return anyResourcesUpdated; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 48d6602..2d83d2f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -474,7 +475,51 @@ private void checkLaunchedContainerNumOnNode(MockRM rm, NodeId nodeId, SchedulerNode node = cs.getSchedulerNode(nodeId); Assert.assertEquals(numContainers, node.getNumContainers()); } - + + @Test + public void testResourceRequestUpdateNodePartitions() throws Exception { + // set node -> label + mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y", false))); + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("y"))); + // inject node label manager + MockRM rm1 = new MockRM(TestUtils.getConfigurationWithQueueLabels(conf)) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // no label + MockNM nm2 = rm1.registerNode("h2:1234", 40 * GB); // label = y + // launch an app to queue b1 (label = y), AM container should be launched in + // nm2 + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2); + List resourceRequest = new ArrayList(); + resourceRequest.add(am1.createResourceReq("*", 1024, 3, 5, "y")); + resourceRequest.add(am1.createResourceReq("h1:1234", 1024, 3, 2, + RMNodeLabelsManager.NO_LABEL)); + resourceRequest.add(am1.createResourceReq("*", 1024, 2, 3, "y")); + resourceRequest.add(am1.createResourceReq("h2:1234", 1024, 2, 4, + RMNodeLabelsManager.NO_LABEL)); + am1.allocate(resourceRequest, new ArrayList()); + CapacityScheduler cs = + (CapacityScheduler) rm1.getRMContext().getScheduler(); + FiCaSchedulerApp app = + cs.getApplicationAttempt(am1.getApplicationAttemptId()); + List allResourceRequests = + app.getAppSchedulingInfo().getAllResourceRequests(); + for (ResourceRequest changeReq : allResourceRequests) { + if (changeReq.getPriority().getPriority() == 2 + || changeReq.getPriority().getPriority() == 3) { + Assert.assertEquals("Expected label y", "y", + changeReq.getNodeLabelExpression()); + } + } + } + @Test public void testPreferenceOfNeedyAppsTowardsNodePartitions() throws Exception { /**