diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index f4a2b0bb261..482e04e1972 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -110,7 +110,6 @@ private final ReadLock maxAllocReadLock; private final WriteLock maxAllocWriteLock; - private boolean useConfiguredMaximumAllocationOnly = true; private long configuredMaximumAllocationWaitTime; protected RMContext rmContext; @@ -216,13 +215,11 @@ public Resource getMinimumResourceCapability() { @Override public Resource getMaximumResourceCapability() { Resource maxResource; + maxAllocReadLock.lock(); try { - if (useConfiguredMaximumAllocationOnly) { - if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() - > configuredMaximumAllocationWaitTime) { - useConfiguredMaximumAllocationOnly = false; - } + if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() + < configuredMaximumAllocationWaitTime) { maxResource = Resources.clone(configuredMaximumAllocation); } else { maxResource = Resources.clone(maximumAllocation); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index dcb1d1f2813..ddbf22b55fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -104,6 +104,32 @@ public void tearDown() { } @Test + public void testSubmitAppWithMaxResReported() throws Exception { + long sleepTime = conf.getLong( + YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, + YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS); + int maxConfMem = conf.getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + + MockRM rm = new MockRM(conf); + rm.start(); + + // a node with half the configured memory + MockNM nm = rm.registerNode("h1:1234", maxConfMem / 2); + //kick the scheduling + nm.nodeHeartbeat(true); + + // sleep this time to timeout the conf value + Thread.sleep(sleepTime); + + // submit app with the max resources reported by the RM + GetNewApplicationResponse resp = rm.getNewAppId(); + rm.submitApp((int) resp.getMaximumResourceCapability().getMemorySize()); + rm.stop(); + } + + @Test public void testGetNewAppId() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index 6a118ac0ab8..c4e27192a0f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -90,6 +90,44 @@ public TestAbstractYarnScheduler(SchedulerType type) { } @Test + public void testInitialGetMaximumResourceCapability() throws Exception { + final Resource nodeResource = Resources.createResource(15*1024, 2); + final Resource configuredMaxResource = Resources.createResource(20*1024, 8); + configureScheduler(); + YarnConfiguration conf = getConf(); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + configuredMaxResource.getVirtualCores()); + conf.setLong(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + configuredMaxResource.getMemorySize()); + conf.setLong( + YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, + 1000); + MockRM rm = new MockRM(conf); + try { + rm.start(); + ResourceScheduler scheduler = rm.getResourceScheduler(); + + Assert.assertEquals(0, scheduler.getNumClusterNodes()); + Resource maximumResourceCapability = scheduler.getMaximumResourceCapability(); + Assert.assertEquals(configuredMaxResource, maximumResourceCapability); + + RMNode node = MockNodes.newNodeInfo( + 0, + nodeResource, 1, "127.0.0.2"); + scheduler.handle(new NodeAddedSchedulerEvent(node)); + Assert.assertEquals(1, scheduler.getNumClusterNodes()); + + // sleep for the timeout + Thread.sleep(1000); + + maximumResourceCapability = scheduler.getMaximumResourceCapability(); + Assert.assertEquals(nodeResource, maximumResourceCapability); + } finally { + rm.stop(); + } + } + + @Test public void testMaximimumAllocationMemory() throws Exception { final int node1MaxMemory = 15 * 1024; final int node2MaxMemory = 5 * 1024;