diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 4f5b4253b06..0f19e38dd1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -949,6 +949,12 @@ private boolean isOverAMShareLimit() { return false; } + private boolean isOverNodeTotalResource(PendingAsk pendingAsk, + FSSchedulerNode node) { + return !Resources.fitsIn(pendingAsk.getPerAllocationResource(), + node.getTotalResource()); + } + @SuppressWarnings("deprecation") private Resource assignContainer(FSSchedulerNode node, boolean reserved) { if (LOG.isTraceEnabled()) { @@ -1012,6 +1018,10 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { + allowedLocality + ", priority: " + schedulerKey.getPriority() + ", app attempt id: " + this.attemptId); } + + if (isOverNodeTotalResource(nodeLocalPendingAsk, node)) { + continue; + } return assignContainer(node, nodeLocalPendingAsk, NodeType.NODE_LOCAL, reserved, schedulerKey); } @@ -1029,6 +1039,9 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { + allowedLocality + ", priority: " + schedulerKey.getPriority() + ", app attempt id: " + this.attemptId); } + if (isOverNodeTotalResource(rackLocalPendingAsk, node)) { + continue; + } return assignContainer(node, rackLocalPendingAsk, NodeType.RACK_LOCAL, reserved, schedulerKey); } @@ -1049,6 +1062,9 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { + schedulerKey.getPriority() + ", app attempt id: " + this.attemptId); } + if (isOverNodeTotalResource(offswitchAsk, node)) { + continue; + } return assignContainer(node, offswitchAsk, NodeType.OFF_SWITCH, reserved, schedulerKey); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 1a6069379a9..219aace5e0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1551,6 +1551,58 @@ public void testContainerReservationAttemptExceedingQueueMax() getCurrentReservation().getMemorySize()); } + @Test (timeout = 500000) + public void testContainerNotAssignOrReserveWhenRequestOverNodeTotalResource() + throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,5vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Add a node + RMNode node1 = + MockNodes + .newNodeInfo(1, Resources.createResource(3072, 5), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + ApplicationAttemptId attId1 = + createSchedulingRequest(3072, 8, "queue1", "user1", 1); + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent); + + // Ensure no container be assigned or reserved when + // request is more than the node's total resource + assertEquals(0, scheduler.getSchedulerApp(attId1). + getLiveContainers().size()); + assertEquals(0, scheduler.getSchedulerApp(attId1). + getCurrentReservation().getMemorySize()); + + // Assign container when request is small than node's total resource + ApplicationAttemptId attId2 = + createSchedulingRequest(1024, 4, "queue1", "user1", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + assertEquals(1, scheduler.getSchedulerApp(attId2). + getLiveContainers().size()); + } + /** * The test verifies that zero-FairShare queues (because of zero/tiny * weight) can get resources for the AM.