diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 46e732ecdb3..2f690b85f45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -230,6 +230,16 @@ public Resource getMaximumResourceCapability(String queueName) { return getMaximumResourceCapability(); } + @VisibleForTesting + public void setForceConfiguredMaxAllocation(boolean flag) { + maxAllocWriteLock.lock(); + try { + useConfiguredMaximumAllocationOnly = flag; + } finally { + maxAllocWriteLock.unlock(); + } + } + protected void initMaximumResourceCapability(Resource maximumAllocation) { maxAllocWriteLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cc9f93ce893..7acde846de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2056,7 +2056,17 @@ public Resource getMaximumResourceCapability(String queueName) { LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } - return ((LeafQueue)queue).getMaximumAllocation(); + + // queue.getMaxAllocation returns *configured* maximum allocation. + // getMaximumResourceCapability() returns maximum allocation considers + // per-node maximum resources. So return (component-wise) min of the two. + + Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation(); + Resource clusterMaxAllocationConsiderNodeMax = + getMaximumResourceCapability(); + + return Resources.componentwiseMin(queueMaxAllocation, + clusterMaxAllocationConsiderNodeMax); } private String handleMoveToPlanQueue(String targetQueueName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index aa63e6c0774..24b5468daf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; @@ -70,6 +71,7 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; public class TestContainerAllocation { @@ -736,4 +738,60 @@ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit() rm1.close(); } + + @Test(timeout = 60000) + public void testContainerRejectionWhenAskBeyondDynamicMax() + throws Exception { + CapacitySchedulerConfiguration newConf = + (CapacitySchedulerConfiguration) TestUtils + .getConfigurationWithMultipleQueues(conf); + newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + + MAXIMUM_ALLOCATION_MB, "4096"); + + MockRM rm1 = new MockRM(newConf); + rm1.start(); + + // before any node registered or before registration timeout, + // submit an app beyond queue max leads to failure. + boolean submitFailed = false; + MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1); + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + try { + am1.allocate("*", 5 * GB, 1, null); + } catch (InvalidResourceRequestException e) { + submitFailed = true; + } + Assert.assertTrue(submitFailed); + + // Ask 3GB should also fail because node max allocation is 2GB. + try { + am1.allocate("*", 3 * GB, 1, null); + Assert.fail("expect a failure because request exceeds" + + " max node allocation"); + } catch (Exception e) { + Assert.assertTrue(e instanceof InvalidResourceRequestException); + } + + // Add a new node, now the cluster maximum should be refreshed to 3GB. + CapacityScheduler cs = (CapacityScheduler)rm1.getResourceScheduler(); + cs.setForceConfiguredMaxAllocation(false); + rm1.registerNode("h2:1234", 3 * GB, 1); + + // Now ask 4 GB still fails + submitFailed = false; + try { + am1.allocate("*", 4 * GB, 1, null); + } catch (InvalidResourceRequestException e) { + submitFailed = true; + } + Assert.assertTrue(submitFailed); + + // But ask 3 GB succeeded. + am1.allocate("*", 3 * GB, 1, null); + + rm1.close(); + } }