diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index fb0c5b6..fd1e007 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1249,7 +1249,7 @@ private static long computeSlotMillis(TaskAttemptImpl taskAttempt) { //This is trying to simulate Hadoop one slots, using default //minimum (in theory we should not be doing this) int minSlotMemSize = - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; int simSlotsRequired = minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fa1e740..eb9c283 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -107,13 +107,13 @@ public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + DEFAULT_RM_SCHEDULER_PORT; - /** Miniumum request grant-able by the RM scheduler. */ - public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = - YARN_PREFIX + "scheduler.minimum-allocation-mb"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; - public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = - YARN_PREFIX + "scheduler.minimum-allocation-vcores"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; + /** Increment request grant-able by the RM scheduler. */ + public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB = + YARN_PREFIX + "scheduler.increment-allocation-mb"; + public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB = 1024; + public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = + YARN_PREFIX + "scheduler.increment-allocation-vcores"; + public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1; /** Maximum request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = @@ -122,7 +122,7 @@ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = YARN_PREFIX + "scheduler.maximum-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; - + /** Number of threads to handle scheduler interface.*/ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = RM_PREFIX + "scheduler.client.thread-count"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 43907f0..c7a2a72 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -51,7 +51,7 @@ @BeforeClass public static void setup() throws InterruptedException, IOException { LOG.info("Starting up YARN cluster"); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); if (yarnCluster == null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java index 6ab474d..addea0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java @@ -48,7 +48,7 @@ @BeforeClass public static void setup() throws InterruptedException, IOException { LOG.info("Starting up YARN cluster"); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster( TestUnmanagedAMLauncher.class.getSimpleName(), 1, 1, 1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index d71d193..55f9af3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -200,10 +200,10 @@ - The minimum allocation for every container request at the RM, + The increment allocation for every container request at the RM, in MBs. Memory requests lower than this won't take effect, - and the specified value will get allocated at minimum. - yarn.scheduler.minimum-allocation-mb + and the specified value will get allocated at increment. + yarn.scheduler.increment-allocation-mb 1024 @@ -216,10 +216,10 @@ - The minimum allocation for every container request at the RM, + The increment allocation for every container request at the RM, in terms of virtual CPU cores. Requests lower than this won't take effect, - and the specified value will get allocated the minimum. - yarn.scheduler.minimum-allocation-vcores + and the specified value will get allocated the increment. + yarn.scheduler.increment-allocation-vcores 1 @@ -228,7 +228,7 @@ in terms of virtual CPU cores. Requests higher than this won't take effect, and will get capped to this value. yarn.scheduler.maximum-allocation-vcores - 32 + 4 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 057e1ca..aa60566 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -492,7 +492,7 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id, WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId() .toString(), nodeId); assertEquals("totalMemoryNeededMB wrong", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, totalMemoryNeededMB); String shortLink = ujoin("containerlogs", cont.getContainerId().toString(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 47f8345..a74d2ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -340,39 +340,39 @@ protected static void validateConfigs(Configuration conf) { } // validate scheduler memory allocation setting - int minMem = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int incrMem = conf.getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); int maxMem = conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - if (minMem <= 0 || minMem > maxMem) { + if (incrMem <= 0 || incrMem > maxMem) { throw new YarnRuntimeException("Invalid resource scheduler memory" + " allocation configuration" - + ", " + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB - + "=" + minMem + + ", " + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB + + "=" + incrMem + ", " + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB - + "=" + maxMem + ", min and max should be greater than 0" - + ", max should be no smaller than min."); + + "=" + maxMem + ", increment and max should be greater than 0" + + ", max should be no smaller than increment."); } // validate scheduler vcores allocation setting - int minVcores = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + int incrVcores = conf.getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); int maxVcores = conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - if (minVcores <= 0 || minVcores > maxVcores) { + if (incrVcores <= 0 || incrVcores > maxVcores) { throw new YarnRuntimeException("Invalid resource scheduler vcores" + " allocation configuration" - + ", " + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES - + "=" + minVcores + + ", " + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES + + "=" + incrVcores + ", " + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + "=" + maxVcores + ", min and max should be greater than 0" - + ", max should be no smaller than min."); + + ", increment should be no smaller than increment."); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 2577684..2491aa5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.Node; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.yarn.YarnRuntimeException; import org.apache.hadoop.yarn.api.records.NodeId; @@ -78,8 +77,8 @@ private static final NodeHeartbeatResponse shutDown = recordFactory .newRecordInstance(NodeHeartbeatResponse.class); - private int minAllocMb; - private int minAllocVcores; + private int incrAllocMb; + private int incrAllocVcores; static { resync.setNodeAction(NodeAction.RESYNC); @@ -115,12 +114,12 @@ public synchronized void init(Configuration conf) { + " should be larger than 0."); } - minAllocMb = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - minAllocVcores = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + incrAllocMb = conf.getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); + incrAllocVcores = conf.getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); super.init(conf); } @@ -184,12 +183,12 @@ public RegisterNodeManagerResponse registerNodeManager( return response; } - // Check if this node has minimum allocations - if (capability.getMemory() < minAllocMb - || capability.getVirtualCores() < minAllocVcores) { + // Check if this node has at least one increment allocation + if (capability.getMemory() < incrAllocMb + || capability.getVirtualCores() < incrAllocVcores) { String message = "NodeManager from " + host - + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + + " doesn't satisfy the minimum allocations, Sending SHUTDOWN" + " signal to the NodeManager."; LOG.info(message); response.setDiagnosticsMessage(message); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java index 5a691af..57d82dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java @@ -54,19 +54,17 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - minimumResource.getMemory()), - maximumResource.getMemory()); + Resource maximumResource, Resource stepFactor) { + int normalizedMemory = normalizeInt(r.getMemory(), + minimumResource.getMemory(), maximumResource.getMemory(), + stepFactor.getMemory()); return Resources.createResource(normalizedMemory); } @Override - public Resource roundUp(Resource r, Resource minimumResource) { + public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(),minimumResource.getMemory()) + roundUp(r.getMemory(), stepFactor.getMemory()) ); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java index 2f66990..eaa7370 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java @@ -124,26 +124,22 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - minimumResource.getMemory()), - maximumResource.getMemory()); - int normalizedCores = Math.min( - roundUp( - Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), - minimumResource.getVirtualCores()), - maximumResource.getVirtualCores()); + Resource maximumResource, Resource stepFactor) { + int normalizedMemory = normalizeInt(r.getMemory(), + minimumResource.getMemory(), maximumResource.getMemory(), + stepFactor.getMemory()); + int normalizedCores = normalizeInt(r.getVirtualCores(), + minimumResource.getVirtualCores(), maximumResource.getVirtualCores(), + stepFactor.getVirtualCores()); return Resources.createResource(normalizedMemory, normalizedCores); } @Override - public Resource roundUp(Resource r, Resource minimumResource) { + public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(), minimumResource.getMemory()), - roundUp(r.getVirtualCores(), minimumResource.getVirtualCores()) + roundUp(r.getMemory(), stepFactor.getMemory()), + roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()) ); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java index b2dd19b..ad4fead 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java @@ -51,6 +51,22 @@ public static int roundDown(int a, int b) { return (a / b) * b; } + public static int normalizeInt(int value, int min, int max, int increment) { + if (value > min) { + if (value < max) { + value = roundUp(value, increment); + if (value > max) { + value = max; + } + } else { + value = max; + } + } else { + value = min; + } + return value; + } + /** * Compute the number of containers which can be allocated given * available and required resources. @@ -94,10 +110,11 @@ public abstract Resource multiplyAndNormalizeDown( * @param r resource * @param minimumResource step-factor * @param maximumResource the upper bound of the resource to be allocated + * @param stepFactor the increment for resources to be allocated * @return normalized resource */ public abstract Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource); + Resource maximumResource, Resource stepFactor); /** * Round-up resource r given factor stepFactor. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java index 58cd676..5e67098 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java @@ -132,9 +132,9 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) { } public static Resource normalize( - ResourceCalculator calculator, Resource lhs, Resource factor, - Resource limit) { - return calculator.normalize(lhs, factor, limit); + ResourceCalculator calculator, Resource lhs, Resource min, + Resource max, Resource increment) { + return calculator.normalize(lhs, min, max, increment); } public static Resource roundUp( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index e4e15b3..8b23def 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -91,11 +91,12 @@ public static void normalizeRequests( ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, - Resource maximumResource) { + Resource maximumResource, + Resource incrementResource) { for (ResourceRequest ask : asks) { normalizeRequest( ask, resourceCalculator, clusterResource, minimumResource, - maximumResource); + maximumResource, incrementResource); } } @@ -108,11 +109,12 @@ public static void normalizeRequest( ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, - Resource maximumResource) { + Resource maximumResource, + Resource incrementResource) { Resource normalized = Resources.normalize( resourceCalculator, ask.getCapability(), minimumResource, - maximumResource); + maximumResource, incrementResource); ask.setCapability(normalized); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 08f667c..79fff28 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -80,6 +80,14 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, public Resource getMaximumResourceCapability(); /** + * Get increment allocatable {@link Resource}. + * @return increment allocatable resource + */ + @Public + @Stable + public Resource getIncrementResourceCapability(); + + /** * Get the number of nodes available in the cluster. * @return the number of available nodes. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 4ebf94b..dd2c3af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,6 +137,7 @@ public Configuration getConf() { private Resource minimumAllocation; private Resource maximumAllocation; + private Resource incrementAllocation; private Map applications = new ConcurrentHashMap(); @@ -177,6 +178,11 @@ public Resource getMaximumResourceCapability() { } @Override + public Resource getIncrementResourceCapability() { + return incrementAllocation; + } + + @Override public Comparator getApplicationComparator() { return applicationComparator; } @@ -214,6 +220,7 @@ public Resource getClusterResources() { this.minimumAllocation = this.conf.getMinimumAllocation(); this.maximumAllocation = this.conf.getMaximumAllocation(); + this.incrementAllocation = this.conf.getIncrementAllocation(); this.calculator = this.conf.getResourceCalculator(); this.rmContext = rmContext; @@ -224,6 +231,7 @@ public Resource getClusterResources() { LOG.info("Initialized CapacityScheduler with " + "calculator=" + getResourceCalculator().getClass() + ", " + "minimumAllocation=<" + getMinimumResourceCapability() + ">, " + + "incrementAllocation=<" + getIncrementResourceCapability() + ">" + "maximumAllocation=<" + getMaximumResourceCapability() + ">"); } else { @@ -485,7 +493,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, // Sanity check SchedulerUtils.normalizeRequests( ask, calculator, getClusterResources(), minimumAllocation, - maximumAllocation); + maximumAllocation, incrementAllocation); // Release containers for (ContainerId releasedContainerId : release) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8d71f00..86f96f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -301,13 +301,7 @@ public void setQueues(String queue, String[] subQueues) { } public Resource getMinimumAllocation() { - int minimumMemory = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - int minimumCores = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); - return Resources.createResource(minimumMemory, minimumCores); + return getIncrementAllocation(); } public Resource getMaximumAllocation() { @@ -320,6 +314,16 @@ public Resource getMaximumAllocation() { return Resources.createResource(maximumMemory, maximumCores); } + public Resource getIncrementAllocation() { + int incrementMemory = getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); + int incrementCores = getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); + return Resources.createResource(incrementMemory, incrementCores); + } + public boolean getEnableUserMetrics() { return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b996952..144f376 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -115,6 +115,7 @@ private RMContext rmContext; private Resource minimumAllocation; private Resource maximumAllocation; + private Resource incrAllocation; private QueueManager queueMgr; private Clock clock; @@ -261,7 +262,7 @@ private void updatePreemptionVariables() { */ boolean isStarvedForMinShare(FSLeafQueue sched) { Resource desiredShare = Resources.min(RESOURCE_CALCULATOR, clusterCapacity, - sched.getMinShare(), sched.getDemand()); + sched.getMinShare(), sched.getDemand()); return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity, sched.getResourceUsage(), desiredShare); } @@ -526,6 +527,11 @@ public Resource getMaximumResourceCapability() { return maximumAllocation; } + @Override + public Resource getIncrementResourceCapability() { + return incrAllocation; + } + public double getNodeLocalityThreshold() { return nodeLocalityThreshold; } @@ -730,7 +736,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // Sanity check SchedulerUtils.normalizeRequests(ask, new DominantResourceCalculator(), - clusterCapacity, minimumAllocation, maximumAllocation); + clusterCapacity, minimumAllocation, maximumAllocation, incrAllocation); // Release containers for (ContainerId releasedContainerId : release) { @@ -988,6 +994,7 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) this.conf = new FairSchedulerConfiguration(conf); minimumAllocation = this.conf.getMinimumAllocation(); maximumAllocation = this.conf.getMaximumAllocation(); + incrAllocation = this.conf.getIncrementAllocation(); userAsDefaultQueue = this.conf.getUserAsDefaultQueue(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 52f0d8f..62be05e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -83,13 +83,7 @@ public FairSchedulerConfiguration(Configuration conf) { } public Resource getMinimumAllocation() { - int mem = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - int cpu = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); - return Resources.createResource(mem, cpu); + return getIncrementAllocation(); } public Resource getMaximumAllocation() { @@ -102,6 +96,16 @@ public Resource getMaximumAllocation() { return Resources.createResource(mem, cpu); } + public Resource getIncrementAllocation() { + int incrementMemory = getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); + int incrementCores = getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); + return Resources.createResource(incrementMemory, incrementCores); + } + public boolean getUserAsDefaultQueue() { return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index e7a2e94..7e29ebd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -109,7 +109,8 @@ private boolean initialized; private Resource minimumAllocation; private Resource maximumAllocation; - + private Resource incrementAllocation; + private Map applications = new TreeMap(); @@ -196,21 +197,27 @@ public Resource getMaximumResourceCapability() { } @Override + public Resource getIncrementResourceCapability() { + return incrementAllocation; + } + + @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { setConf(conf); if (!this.initialized) { this.rmContext = rmContext; - this.minimumAllocation = - Resources.createResource(conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); this.maximumAllocation = Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); - this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, + this.incrementAllocation = + Resources.createResource(conf.getInt( + YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB)); + this.minimumAllocation = this.incrementAllocation; + this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); this.activeUsersManager = new ActiveUsersManager(metrics); this.initialized = true; @@ -232,7 +239,8 @@ public Allocation allocate( // Sanity check SchedulerUtils.normalizeRequests(ask, resourceCalculator, - clusterResource, minimumAllocation, maximumAllocation); + clusterResource, minimumAllocation, maximumAllocation, + incrementAllocation); // Release containers for (ContainerId releasedContainer : release) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 71c2fe0..d7d6565 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -479,7 +479,7 @@ private static ResourceScheduler mockResourceScheduler() { ResourceScheduler scheduler = mock(ResourceScheduler.class); when(scheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB)); when(scheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); @@ -496,7 +496,7 @@ private static ContainerLaunchContext mockContainerLaunchContext( private static Resource mockResource() { return Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index d819467..27fcbe6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -385,7 +385,7 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); @@ -450,7 +450,7 @@ private static YarnScheduler mockYarnScheduler() { YarnScheduler yarnScheduler = mock(YarnScheduler.class); when(yarnScheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB)); when(yarnScheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 160d8d4..f464387 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -177,8 +177,8 @@ private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) nm1.getNodeId()); int checkAlloc = - conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + conf.getInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory()); rm.stop(); @@ -189,7 +189,7 @@ public void testDefaultMinimumAllocation() throws Exception { // Test with something lesser than default testMinimumAllocation( new YarnConfiguration(TestFifoScheduler.conf), - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB / 2); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB / 2); } @Test @@ -197,7 +197,7 @@ public void testNonDefaultMinimumAllocation() throws Exception { // Set custom min-alloc to test tweaking it int allocMB = 1536; YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, allocMB); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, allocMB * 10); // Test for something lesser than this. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index ffe8c81..26b5402 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -207,7 +207,7 @@ public void testResourceManagerInitConfigValidation() throws Exception { } conf = new YarnConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { resourceManager.init(conf); @@ -221,7 +221,7 @@ public void testResourceManagerInitConfigValidation() throws Exception { } conf = new YarnConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { resourceManager.init(conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index e34a40d..e7ff588 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -289,8 +289,8 @@ public void testSetRMIdentifierInRegistration() throws Exception { @Test public void testNodeRegistrationWithMinimumAllocations() throws Exception { Configuration conf = new Configuration(); - conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048"); - conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4"); + conf.set(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, "2048"); + conf.set(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, "4"); rm = new MockRM(conf); rm.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 2227948..c9718f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -69,45 +69,47 @@ public void testNormalizeRequest() { final int minMemory = 1024; final int maxMemory = 8192; + final int incrMemory = 256; Resource minResource = Resources.createResource(minMemory, 0); Resource maxResource = Resources.createResource(maxMemory, 0); + Resource incrResource = Resources.createResource(incrMemory, 0); ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory ask.setCapability(Resources.createResource(-1024)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(minMemory, ask.getCapability().getMemory()); // case zero memory ask.setCapability(Resources.createResource(0)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(minMemory, ask.getCapability().getMemory()); // case memory is a multiple of minMemory ask.setCapability(Resources.createResource(2 * minMemory)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(2 * minMemory, ask.getCapability().getMemory()); // case memory is not a multiple of minMemory ask.setCapability(Resources.createResource(minMemory + 10)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); - assertEquals(2 * minMemory, ask.getCapability().getMemory()); + maxResource, incrResource); + assertEquals(minMemory + incrMemory, ask.getCapability().getMemory()); // case memory is equal to max allowed ask.setCapability(Resources.createResource(maxMemory)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(maxMemory, ask.getCapability().getMemory()); // case memory is just less than max ask.setCapability(Resources.createResource(maxMemory - 10)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(maxMemory, ask.getCapability().getMemory()); // max is not a multiple of min @@ -115,14 +117,14 @@ public void testNormalizeRequest() { ask.setCapability(Resources.createResource(maxMemory - 100)); // multiple of minMemory > maxMemory, then reduce to maxMemory SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); // ask is more than max maxResource = Resources.createResource(maxMemory, 0); ask.setCapability(Resources.createResource(maxMemory + 100)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, incrResource); assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); } @@ -132,6 +134,7 @@ public void testNormalizeRequestWithDominantResourceCalculator() { Resource minResource = Resources.createResource(1024, 1); Resource maxResource = Resources.createResource(10240, 10); + Resource incrResource = Resources.createResource(256, 2); Resource clusterResource = Resources.createResource(10 * 1024, 10); ResourceRequest ask = new ResourceRequestPBImpl(); @@ -139,24 +142,27 @@ public void testNormalizeRequestWithDominantResourceCalculator() { // case negative memory/vcores ask.setCapability(Resources.createResource(-1024, -1)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, clusterResource, minResource, maxResource, + incrResource); assertEquals(minResource, ask.getCapability()); // case zero memory/vcores ask.setCapability(Resources.createResource(0, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, clusterResource, minResource, maxResource, + incrResource); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemory()); // case non-zero memory & zero cores - ask.setCapability(Resources.createResource(1536, 0)); + ask.setCapability(Resources.createResource(1200, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); - assertEquals(Resources.createResource(2048, 1), ask.getCapability()); + ask, resourceCalculator, clusterResource, minResource, maxResource, + incrResource); + assertEquals(Resources.createResource(1280, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); - assertEquals(2048, ask.getCapability().getMemory()); + assertEquals(1280, ask.getCapability().getMemory()); } @Test (timeout = 30000) @@ -169,7 +175,7 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( 0, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.validateResourceRequest(resReq, maxResource); @@ -180,7 +186,7 @@ public void testValidateResourceRequest() { // zero vcores try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 0); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -193,7 +199,7 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.validateResourceRequest(resReq, maxResource); @@ -204,7 +210,7 @@ public void testValidateResourceRequest() { // max vcores try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -217,7 +223,7 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( -1, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.validateResourceRequest(resReq, maxResource); @@ -229,7 +235,7 @@ public void testValidateResourceRequest() { // negative vcores try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, -1); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -243,7 +249,7 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.validateResourceRequest(resReq, maxResource); @@ -255,7 +261,7 @@ public void testValidateResourceRequest() { // more than max vcores try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); ResourceRequest resReq = BuilderUtils.newResourceRequest( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index bfda5e5..4c778b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -116,7 +116,7 @@ public void tick(int seconds) { public void setUp() throws IOException { scheduler = new FairScheduler(); Configuration conf = createConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); @@ -235,7 +235,8 @@ public void testLoadConfigurationOnInitialize() throws IOException { conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, .5); conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); scheduler.reinitialize(conf, resourceManager.getRMContext()); Assert.assertEquals(true, scheduler.assignMultiple); Assert.assertEquals(3, scheduler.maxAssign); @@ -244,6 +245,7 @@ public void testLoadConfigurationOnInitialize() throws IOException { Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01); Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory()); Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(128, scheduler.getIncrementResourceCapability().getMemory()); } @Test @@ -367,8 +369,8 @@ public void testSimpleContainerAllocation() { NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); - // Asked for less than min_allocation. - assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + // Asked for less than increment allocation. + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemory()); @@ -521,7 +523,8 @@ public void testQueueDemandCalculation() throws Exception { ApplicationAttemptId id22 = createAppAttemptId(2, 2); scheduler.addApplication(id22, "root.queue2", "user1"); - int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; + int minReqSize = + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 12ed779..bc24645 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -20,9 +20,16 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; +import junit.framework.Assert; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; +import java.io.IOException; + public class TestFairSchedulerConfiguration { @Test public void testParseResourceConfigValue() throws Exception { @@ -55,4 +62,5 @@ public void testOnlyCPU() throws Exception { public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 5af6585..98bbdff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; @@ -114,6 +115,22 @@ private ResourceRequest createResourceRequest(int memory, String host, return request; } + @Test + public void testMinMaxIncrResourcesSettings() throws IOException { + FifoScheduler cs = new FifoScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + cs.setConf(conf); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(128, cs.getIncrementResourceCapability().getMemory()); + } + @Test(timeout=5000) public void testFifoSchedulerCapacityWhenNoNMs() { FifoScheduler scheduler = new FifoScheduler(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index f07becb..aa7d8f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -547,7 +547,7 @@ public void verifyClusterSchedulerFifoGeneric(String type, String state, assertEquals("usedCapacity doesn't match", 0.0, usedCapacity, 0.0); assertEquals( "minQueueMemoryCapacity doesn't match", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, minQueueCapacity); assertEquals("maxQueueMemoryCapacity doesn't match", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,