Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (working copy) @@ -229,6 +229,12 @@ | | capacity irrespective of how idle th cluster is. Value is specified as | | | a float.| *--------------------------------------+--------------------------------------+ +| <<.maximum-allocation-mb>>> | | +| | The per queue maximum limit of memory to allocate to each container | +| | request at the Resource Manager. This setting overrides the cluster | +| | configuration <<>>. This value | +| | must be smaller than or equal to the cluster maximum. | +*--------------------------------------+--------------------------------------+ * Running and Pending Application Limits Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (working copy) @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.util.List; @@ -241,7 +243,7 @@ conf.setUserLimitFactor(B2, 100.0f); conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); - + LOG.info("Setup top-level queues a and b"); } @@ -273,7 +275,193 @@ cs.reinitialize(conf,null); checkQueueCapacities(cs, 80f, 20f); } + + @Test + public void testRefreshQueuesMaxAllocationRefresh() throws Exception { + // queue refresh should not allow changing the maximum allocation setting + // per queue to be smaller than previous setting + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + assertEquals("max allocation in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemory()); + assertEquals("max allocation for A1", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + conf.getMaximumAllocationMbPerQueue(A1).getMemory()); + assertEquals("max allocation", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + conf.getMaximumAllocation().getMemory()); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueA1 = findQueue(queueA, A1); + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemory(), 8192); + + conf.setMaximumAllocation(A1, 4096); + + try { + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + fail("should have thrown exception"); + } catch (IOException e) { + + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { + // verify we can't set the allocation per queue larger then cluster setting + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + // change max allocation for B3 queue to be larger then cluster max + conf.setMaximumAllocation(B3, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048); + cs.setConf(new YarnConfiguration()); + try { + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + fail("should have thrown exception"); + } catch (IllegalArgumentException e) { + assertTrue("max allocation exception", + e.getMessage().contains("max allocation")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { + // queue refresh should allow max allocation per queue to go larger + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + conf.setMaximumAllocation(A1, 4096); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + + assertEquals("max allocation in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemory()); + assertEquals("max allocation", + 4096, + conf.getMaximumAllocationMbPerQueue(A1).getMemory()); + assertEquals("max allocation", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + conf.getMaximumAllocation().getMemory()); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueA1 = findQueue(queueA, A1); + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemory(), 4096); + + conf.setMaximumAllocation(A1, 6144); + cs.reinitialize(conf,null); + // conf will have changed but we shouldn't be able to change max allocation + // for the actual queue + assertEquals("max allocation", 6144, conf + .getMaximumAllocationMbPerQueue(A1).getMemory()); + assertEquals("max allocation", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, conf + .getMaximumAllocation().getMemory()); + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemory(), 6144); + assertEquals("max allocation in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemory()); + } + + @Test + public void testRefreshQueuesMaxAllocationCSError() throws Exception { + // Try to refresh the cluster level max allocation size to be smaller + // and it should error out + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + conf.setMaximumAllocation(10240); + conf.setMaximumAllocation(A1, 4096); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + + assertEquals("max allocation in CS", 10240, cs + .getMaximumResourceCapability().getMemory()); + + conf.setMaximumAllocation(6144); + try { + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { + // Try to refresh the cluster level max allocation size to be larger + // and verify that if there is no setting per queue it uses the + // cluster level setting. + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + conf.setMaximumAllocation(10240); + conf.setMaximumAllocation(A1, 4096); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + + assertEquals("max allocation in CS", 10240, cs + .getMaximumResourceCapability().getMemory()); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueA1 = findQueue(queueA, A1); + CSQueue queueA2 = findQueue(queueA, A2); + CSQueue queueB2 = findQueue(queueB, B2); + + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemory(), 4096); + assertEquals("queue max allocation", ((LeafQueue) queueA2) + .getMaximumAllocation().getMemory(), 10240); + assertEquals("queue max allocation", ((LeafQueue) queueB2) + .getMaximumAllocation().getMemory(), 10240); + + conf.setMaximumAllocation(12288); + cs.reinitialize(conf,null); + // cluster level setting should change and any queues without + // per queue setting + assertEquals("max allocation in CS", 12288, cs + .getMaximumResourceCapability().getMemory()); + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemory(), 4096); + assertEquals("queue max allocation", ((LeafQueue) queueA2) + .getMaximumAllocation().getMemory(), 12288); + assertEquals("queue max allocation", ((LeafQueue) queueB2) + .getMaximumAllocation().getMemory(), 12288); + } + private void checkQueueCapacities(CapacityScheduler cs, float capacityA, float capacityB) { CSQueue rootQueue = cs.getRootQueue(); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (working copy) @@ -194,7 +194,7 @@ response.setMinimumResourceCapability(rScheduler .getMinimumResourceCapability()); response.setMaximumResourceCapability(rScheduler - .getMaximumResourceCapability()); + .getMaximumResourceCapability(app.getQueue())); response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId) .getSubmissionContext().getAMContainerSpec().getApplicationACLs()); return response; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (working copy) @@ -191,6 +191,11 @@ public Resource getMaximumResourceCapability() { return maximumAllocation; } + + @Override + public Resource getMaximumResourceCapability(String queueName) { + return maximumAllocation; + } @Override public synchronized void Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (working copy) @@ -71,15 +71,28 @@ @Stable public Resource getMinimumResourceCapability(); + /** - * Get maximum allocatable {@link Resource}. + * Get maximum allocatable {@link Resource} + * at the cluster level. * @return maximum allocatable resource */ @Public @Stable public Resource getMaximumResourceCapability(); + /** + * Get maximum allocatable {@link Resource} for the + * queue specified. + * @param queueName queue name + * @return maximum allocatable resource + */ + @Public + @Stable + public Resource getMaximumResourceCapability(String queueName); + + /** * Get the number of nodes available in the cluster. * @return the number of available nodes. */ Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java (working copy) @@ -32,6 +32,8 @@ Resource getMinimumResourceCapability(); Resource getMaximumResourceCapability(); + + Resource getMaximumResourceCapability(String queueName); RMContainerTokenSecretManager getContainerTokenSecretManager(); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (working copy) @@ -79,6 +79,9 @@ @Private public static final String STATE = "state"; + + @Private + public static final String MAXIMUM_ALLOCATION_MB = "maximum-allocation-mb"; @Private public static final int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS = 10000; @@ -292,13 +295,67 @@ return Resources.createResource(minimumMemory); } + /** + * Get the cluster level setting for the maximum limit of memory to allocate + * to each container request at the Resource Manager. + * + * @return setting specified or defaults to + * YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + */ public Resource getMaximumAllocation() { int maximumMemory = getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); return Resources.createResource(maximumMemory); } + + /** + * Get the per queue setting for the maximum limit of memory to allocate to + * each container request. + * + * @param queue + * name of the queue + * @return setting specified per queue else falls back to the cluster setting + */ + public Resource getMaximumAllocationMbPerQueue(String queue) { + int maxAllocationMbPerQueue = + getInt(getQueuePrefix(queue) + MAXIMUM_ALLOCATION_MB, + (int)UNDEFINED); + Resource clusterMax = getMaximumAllocation(); + LOG.debug("max alloc mb per queue for: " + queue + " is: " + maxAllocationMbPerQueue); + if (maxAllocationMbPerQueue == (int)UNDEFINED) { + LOG.info("max alloc mb per queue for: " + queue + " is undefined"); + return clusterMax; + } + if (maxAllocationMbPerQueue > clusterMax.getMemory()) { + throw new IllegalArgumentException( + "Queue maximum allocation MB cannot be larger than the cluster setting for queue: " + + queue + + " max allocation per queue: " + + maxAllocationMbPerQueue + + " cluster setting: " + clusterMax.getMemory()); + } + return Resources.createResource(maxAllocationMbPerQueue); + } + + /* + * For testing only + */ + void setMaximumAllocation(int max) { + setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, max); + LOG.debug("CSConf - setMaxAllocationMB for CS, max=" + max); + } + + /* + * For testing only + */ + void setMaximumAllocation(String queue, int max) { + setInt(getQueuePrefix(queue) + MAXIMUM_ALLOCATION_MB, max); + LOG.debug("CSConf - setMaxAllocationMB: queuePrefix=" + getQueuePrefix(queue) + + ", max=" + max); + } + public boolean getEnableUserMetrics() { return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (working copy) @@ -170,13 +170,27 @@ public Resource getMinimumResourceCapability() { return minimumAllocation; } - + @Override public Resource getMaximumResourceCapability() { return maximumAllocation; } @Override + public Resource getMaximumResourceCapability(String queueName) { + CSQueue queue = getQueue(queueName); + if (queue == null) { + LOG.error("queue: " + queueName + " is null"); + return maximumAllocation; + } + if (!(queue instanceof LeafQueue)) { + LOG.error("queue: " + queueName + "is not a leafQueue"); + return maximumAllocation; + } + return ((LeafQueue)queue).getMaximumAllocation(); + } + + @Override public synchronized int getNumClusterNodes() { return numNodeManagers; } @@ -206,6 +220,16 @@ CapacitySchedulerConfiguration oldConf = this.conf; this.conf = new CapacitySchedulerConfiguration(conf); try { + if (this.conf.getMaximumAllocation().getMemory() < this.maximumAllocation + .getMemory()) { + throw new IOException( + "Trying to reinitialize capacity scheduler," + + " the maximum allocation size in mb can not be decreased! Current setting: " + + this.maximumAllocation.getMemory() + ", trying to set it to: " + + this.conf.getMaximumAllocation().getMemory()); + } + this.maximumAllocation = this.conf.getMaximumAllocation(); + LOG.info("Re-initializing queues..."); reinitializeQueues(this.conf); } catch (Throwable t) { Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (revision 1560805) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (working copy) @@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; @@ -101,8 +100,8 @@ Set pendingApplications; private final Resource minimumAllocation; - private final Resource maximumAllocation; - private final float minimumAllocationFactor; + private Resource maximumAllocation; + private float minimumAllocationFactor; private RMContainerTokenSecretManager containerTokenSecretManager; @@ -138,11 +137,7 @@ cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); this.activeUsersManager = new ActiveUsersManager(metrics); - this.minimumAllocation = cs.getMinimumResourceCapability(); - this.maximumAllocation = cs.getMaximumResourceCapability(); - this.minimumAllocationFactor = - (float)(maximumAllocation.getMemory() - minimumAllocation.getMemory()) / - maximumAllocation.getMemory(); + this.containerTokenSecretManager = cs.getContainerTokenSecretManager(); float capacity = @@ -166,6 +161,8 @@ maxApplicationsPerUser = (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor); + this.minimumAllocation = cs.getMinimumResourceCapability(); + this.maxAMResourcePerQueuePercent = cs.getConfiguration(). getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); @@ -197,8 +194,9 @@ userLimit, userLimitFactor, maxApplications, maxApplicationsPerUser, maxActiveApplications, maxActiveApplicationsPerUser, - state, acls, cs.getConfiguration().getNodeLocalityDelay()); - + state, acls, cs.getConfiguration().getNodeLocalityDelay(), + cs.getConfiguration().getMaximumAllocationMbPerQueue(getQueuePath())); + if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); @@ -217,7 +215,7 @@ int maxApplications, int maxApplicationsPerUser, int maxActiveApplications, int maxActiveApplicationsPerUser, QueueState state, Map acls, - int nodeLocalityDelay) + int nodeLocalityDelay, Resource maxAllocation) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); @@ -248,6 +246,13 @@ this.queueInfo.setQueueState(this.state); this.nodeLocalityDelay = nodeLocalityDelay; + + this.maximumAllocation = maxAllocation; + + // re-init this since max allocation could have changed + this.minimumAllocationFactor = + (float)(maximumAllocation.getMemory() - minimumAllocation.getMemory()) / + maximumAllocation.getMemory(); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -305,6 +310,8 @@ "minimumAllocationFactor = " + minimumAllocationFactor + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + "maximumAllocationMemory ]" + "\n" + + "maximumAllocation = " + maximumAllocation + + " [= configuredMaxAllocation ]" + "\n" + "numContainers = " + numContainers + " [= currentNumContainers ]" + "\n" + "state = " + state + @@ -367,10 +374,6 @@ return minimumAllocation; } - /** - * Used only by tests. - */ - @Private public Resource getMaximumAllocation() { return maximumAllocation; } @@ -585,8 +588,21 @@ throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } + + LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; - LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; + // don't allow the maximum allocation is be decreased in size + // since we have already told running AM's the size + if (newlyParsedLeafQueue.getMaximumAllocation().getMemory() < getMaximumAllocation() + .getMemory()) { + throw new IOException( + "Trying to reinitialize " + + getQueuePath() + + " the maximum allocation size in mb can not be decreased! Current setting: " + + getMaximumAllocation().getMemory() + ", trying to set it to: " + + newlyParsedLeafQueue.getMaximumAllocation().getMemory()); + } + setupQueueConfigs( clusterResource, newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity, @@ -598,7 +614,8 @@ newlyParsedLeafQueue.getMaximumActiveApplications(), newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(), newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls, - newlyParsedLeafQueue.getNodeLocalityDelay()); + newlyParsedLeafQueue.getNodeLocalityDelay(), + newlyParsedLeafQueue.getMaximumAllocation()); } @Override