diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index a398906..690f47a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; @@ -493,7 +494,23 @@ public boolean canRunAppAM(Resource amResource) { if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } - Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); + + // If a queue weight is zero, use maxShare to compute maxAMResource since + // its fair share is always zero. + ResourceWeights weight = scheduler.getAllocationConfiguration(). + getQueueWeight(getName()); + Resource availableResource = Resources.createResource( + scheduler.getRootQueueMetrics().getAvailableMB(), + scheduler.getRootQueueMetrics().getAvailableVirtualCores()); + if (Resources.fitsIn(getMaxShare(), availableResource)) { + availableResource = getMaxShare(); + } + Resource maxResource = + (weight.getWeight(ResourceType.MEMORY) == 0 && + weight.getWeight(ResourceType.CPU) == 0) ? + availableResource : getFairShare(); + + Resource maxAMResource = Resources.multiply(maxResource, maxAMShare); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); boolean overMaxAMShareLimit = policy diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b3931c9..d95a5db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1081,6 +1081,70 @@ public void testContainerReservationAttemptExceedingQueueMax() getCurrentReservation().getMemory()); } + /** + * The test verifies that zero weight queues can get resources no matter + * whether there is active or non-active non-zero weight queues. + */ + @Test + public void testRequestAMResourceInZeroWeightQueue() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.0"); + out.println("4096mb,10vcores"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("2.0"); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node = + MockNodes.newNodeInfo(1, Resources.createResource(8192, 20), + 0, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.update(); + + //create request for non-zero weight queue + createSchedulingRequest(1024, "root.queue2", "user2", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // A managed AM which need 3G memory will not get resource, + // since it request more than the maxAMShare (4G * 0.5 = 2G). + Resource amResource = Resource.newInstance(1024, 1); + int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); + ApplicationAttemptId attId1 = createAppAttemptId(1, 1); + createApplicationWithAMResource(attId1, "root.queue1", "user1", amResource); + createSchedulingRequestExistingApplication(3 * 1024, 1, amPriority, attId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application 1 should not be running", + 0, app1.getLiveContainers().size()); + + // A managed AM which need 2G memory will get resource, + // since it request no more than the maxAMShare (4G * 0.5 = 2G). + ApplicationAttemptId attId2 = createAppAttemptId(2, 1); + createApplicationWithAMResource(attId2, "root.queue1", "user1", amResource); + createSchedulingRequestExistingApplication(2 * 1024, 1, amPriority, attId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application 2 should be running", + 1, app2.getLiveContainers().size()); + } + @Test (timeout = 500000) public void testContainerReservationNotExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);