diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 310b3b5..b2d43c9 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -935,6 +935,16 @@ public Resource getClusterResource() { return null; } + /** + * Get increment allocatable {@link Resource}. + * + * @return incremental allocatable resource. + */ + @Override + public Resource getIncrementResourceCapability() { + return this.getMinimumResourceCapability(); + } + @Override public synchronized List getTransferredContainers( ApplicationAttemptId currentAttempt) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 703ec1e..daaf1b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -410,7 +410,7 @@ private ResourceRequest validateAndCreateResourceRequest( scheduler.getClusterResource(), scheduler.getMinimumResourceCapability(), scheduler.getMaximumResourceCapability(), - scheduler.getMinimumResourceCapability()); + scheduler.getIncrementResourceCapability()); return amReq; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 0c2ae36..ea8c854 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -88,6 +88,13 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, public Resource getClusterResource(); /** + * Get increment allocatable {@link Resource}. + * @return incremental allocatable resource. + */ + @Public + @Stable + public Resource getIncrementResourceCapability(); + /** * Get minimum allocatable {@link Resource}. * @return minimum allocatable resource */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index ee31bea..0e145e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -537,7 +537,7 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer, for (CSQueue queue : queues.values()) { AbstractCSQueue csQueue = (AbstractCSQueue) queue; authorizer.setPermission(csQueue.getPrivilegedEntity(), - csQueue.getACLs(), UserGroupInformation.getCurrentUser()); + csQueue.getACLs(), UserGroupInformation.getCurrentUser()); } } @@ -962,6 +962,16 @@ public QueueInfo getQueueInfo(String queueName, return root.getQueueUserAclInfo(user); } + /** + * Get increment allocatable {@link Resource}. + * + * @return incremental allocatable resource. + */ + @Override + public Resource getIncrementResourceCapability() { + return this.getMinimumResourceCapability(); + } + private synchronized void nodeUpdate(RMNode nm) { if (LOG.isDebugEnabled()) { LOG.debug("nodeUpdate: " + nm + " clusterResources: " + clusterResource); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 2ec2311..cbb1577 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -953,6 +953,16 @@ public QueueInfo getQueueInfo(String queueName, return DEFAULT_QUEUE.getQueueUserAclInfo(null); } + /** + * Get increment allocatable {@link Resource}. + * + * @return incremental allocatable resource. + */ + @Override + public Resource getIncrementResourceCapability() { + return this.getMinimumResourceCapability(); + } + @Override public ResourceCalculator getResourceCalculator() { return resourceCalculator; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index c435692..a8f6e01 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -629,6 +631,29 @@ public void testRMAppSubmitInvalidResourceRequest() throws Exception { } } + @Test(timeout = 30000) + public void testFairNormalizeRequests() throws Exception { + long now = System.currentTimeMillis(); + FairScheduler scheduler = new FairScheduler(); + ((RMContextImpl)rmContext).setScheduler(scheduler); + Configuration conf = new YarnConfiguration(); + + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 2048); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1096); + scheduler.init(conf); + scheduler.reinitialize(conf, null); + + Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(2048, scheduler.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(1096, scheduler.getIncrementResourceCapability().getMemory()); + + //Creating a Mock app to test the Resource allocation + asContext.setResource(Resources.createResource(1024)); + appMonitor.submitApplication(asContext, "YARN-3996_test"); + Assert.assertEquals(1024,asContext.getResource().getMemory()); + } + @Test (timeout = 30000) public void testEscapeApplicationSummary() { RMApp app = mock(RMAppImpl.class);