diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 332b88f..6ed7cd6 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1328,7 +1328,7 @@ private static AppContext createAppContext( when(context.getJob(isA(JobId.class))).thenReturn(job); when(context.getClusterInfo()).thenReturn( new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils - .newResource(10240, 1))); + .newResource(10240, 1), BuilderUtils.newResource(256, 1))); when(context.getEventHandler()).thenReturn(new EventHandler() { @Override public void handle(Event event) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 25849f4..9c05465 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -199,7 +199,8 @@ public void verifySlotMillis(int mapMemMb, int reduceMemMb, conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb); conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb); app.setClusterInfo(new ClusterInfo(BuilderUtils - .newResource(minContainerSize, 1), BuilderUtils.newResource(10240, 1))); + .newResource(minContainerSize, 1), BuilderUtils.newResource(10240, 1), + BuilderUtils.newResource(256, 1))); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java index 6db4816..9103868 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java @@ -118,7 +118,7 @@ private static AppContext createAppContext() { when(ctx.getJob(isA(JobId.class))).thenReturn(job); when(ctx.getClusterInfo()).thenReturn( new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils - .newResource(10240, 1))); + .newResource(10240, 1), BuilderUtils.newResource(256, 1))); when(ctx.getEventHandler()).thenReturn(eventHandler); return ctx; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java index 8b1b709..8a66718 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java @@ -72,6 +72,19 @@ public void setMaximumResourceCapability(Resource capability); /** + * Get the multiplier capability for any {@link Resource} allocated by the + * ResourceManager in the cluster. + * @return multiplier capability of allocated resources in the cluster + */ + @Public + @Stable + public Resource getMultiplierResourceCapability(); + + @Private + @Unstable + public void setMultiplierResourceCapability(Resource capability); + + /** * Get the ApplicationACLs for the application. * @return all the ApplicationACLs */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java index b861282..5037fa9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java @@ -46,6 +46,7 @@ private Resource minimumResourceCapability; private Resource maximumResourceCapability; + private Resource multiplierResourceCapability; private Map applicationACLS = null; public RegisterApplicationMasterResponsePBImpl() { @@ -79,7 +80,11 @@ private void mergeLocalToBuilder() { } if (this.maximumResourceCapability != null) { builder.setMaximumCapability( - convertToProtoFormat(this.maximumResourceCapability)); + convertToProtoFormat(this.maximumResourceCapability)); + } + if (this.multiplierResourceCapability != null) { + builder.setMultiplierCapability( + convertToProtoFormat(this.multiplierResourceCapability)); } if (this.applicationACLS != null) { addApplicationACLs(); @@ -125,6 +130,22 @@ public Resource getMinimumResourceCapability() { } @Override + public Resource getMultiplierResourceCapability() { + if (this.multiplierResourceCapability != null) { + return this.multiplierResourceCapability; + } + + RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasMultiplierCapability()) { + return null; + } + + this.multiplierResourceCapability + = convertFromProtoFormat(p.getMultiplierCapability()); + return this.multiplierResourceCapability; + } + + @Override public void setMaximumResourceCapability(Resource capability) { maybeInitBuilder(); if(maximumResourceCapability == null) { @@ -142,6 +163,14 @@ public void setMinimumResourceCapability(Resource capability) { this.minimumResourceCapability = capability; } + @Override + public void setMultiplierResourceCapability(Resource capability) { + maybeInitBuilder(); + if(multiplierResourceCapability == null) { + builder.clearMultiplierCapability(); + } + this.multiplierResourceCapability = capability; + } @Override public Map getApplicationACLs() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 6ac0274..2db04f1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -38,6 +38,7 @@ message RegisterApplicationMasterResponseProto { optional ResourceProto minimumCapability = 1; optional ResourceProto maximumCapability = 2; repeated ApplicationACLMapProto application_ACLs = 3; + optional ResourceProto multiplierCapability = 4; } message FinishApplicationMasterRequestProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java index 2764061..28333a6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -143,8 +145,11 @@ public void testAMRMClient() throws YarnRemoteException, IOException { amClient.init(conf); amClient.start(); + RegisterApplicationMasterResponse response = amClient.registerApplicationMaster("Host", 10000, ""); + testMinMaxMultiplerResourceValues(response); + testAllocation(amClient); amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, @@ -156,7 +161,27 @@ public void testAMRMClient() throws YarnRemoteException, IOException { } } } - + + private void testMinMaxMultiplerResourceValues( + RegisterApplicationMasterResponse response) + throws YarnRemoteException, IOException { + int minMem = response.getMinimumResourceCapability().getMemory(); + int maxMem = response.getMaximumResourceCapability().getMemory(); + int multMem = response.getMultiplierResourceCapability().getMemory(); + int minCPUs = response.getMinimumResourceCapability().getVirtualCores(); + int maxCPUs = response.getMaximumResourceCapability().getVirtualCores(); + int multCPUs = response.getMultiplierResourceCapability().getVirtualCores(); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minMem); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxMem); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, + multMem); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + minCPUs); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + maxCPUs); + assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES, + multCPUs); + } private void testAllocation(final AMRMClientImpl amClient) throws YarnRemoteException, IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java index 64c7ca7..9cc0c81 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java @@ -24,15 +24,20 @@ public class ClusterInfo { private Resource minContainerCapability; private Resource maxContainerCapability; + private Resource multiplierContainerCapacity; public ClusterInfo() { this.minContainerCapability = Records.newRecord(Resource.class); this.maxContainerCapability = Records.newRecord(Resource.class); + this.multiplierContainerCapacity = Records.newRecord(Resource.class); } - public ClusterInfo(Resource minCapability, Resource maxCapability) { + public ClusterInfo(Resource minCapability, Resource maxCapability, + Resource multiplierContainerCapacity) { this.minContainerCapability = minCapability; this.maxContainerCapability = maxCapability; + this.multiplierContainerCapacity = multiplierContainerCapacity; + } public Resource getMinContainerCapability() { @@ -50,4 +55,14 @@ public Resource getMaxContainerCapability() { public void setMaxContainerCapability(Resource maxContainerCapability) { this.maxContainerCapability = maxContainerCapability; } + + public Resource getMultiplierContainerCapability() { + return multiplierContainerCapacity; + } + + public void setMultiplierContainerCapability( + Resource multiplierContainerCapacity) { + this.multiplierContainerCapacity = multiplierContainerCapacity; + } + } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f9b017d..58d09e4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -126,6 +126,14 @@ YARN_PREFIX + "scheduler.maximum-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; + /** Multiplier request grant-able by the RM scheduler. */ + public static final String RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB = + YARN_PREFIX + "scheduler.multiplier-allocation-mb"; + public static final int DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB = 256; + public static final String RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES = + YARN_PREFIX + "scheduler.multiplier-allocation-vcores"; + public static final int DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES = 1; + /** Number of threads to handle scheduler interface.*/ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = RM_PREFIX + "scheduler.client.thread-count"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 599f8a9..49f1c69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -228,7 +228,7 @@ in terms of virtual CPU cores. Requests higher than this won't take effect, and will get capped to this value. yarn.scheduler.maximum-allocation-vcores - 32 + 4 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 3094a93..c8e1e0a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -211,6 +211,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster( .getMinimumResourceCapability()); response.setMaximumResourceCapability(rScheduler .getMaximumResourceCapability()); + response.setMultiplierResourceCapability(rScheduler + .getMultiplierResourceCapability()); response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId) .getSubmissionContext().getAMContainerSpec().getApplicationACLs()); return response; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java index 5a691af..65191f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java @@ -54,19 +54,17 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - minimumResource.getMemory()), - maximumResource.getMemory()); + Resource maximumResource, Resource stepFactor) { + int normalizedMemory = normalizeInt(r.getMemory(), + minimumResource.getMemory(), maximumResource.getMemory(), + stepFactor.getMemory()); return Resources.createResource(normalizedMemory); } @Override - public Resource roundUp(Resource r, Resource minimumResource) { + public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(),minimumResource.getMemory()) + roundUp(r.getMemory(), stepFactor.getMemory()) ); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java index 2f66990..ddf8f00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java @@ -124,26 +124,22 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - int normalizedMemory = Math.min( - roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - minimumResource.getMemory()), - maximumResource.getMemory()); - int normalizedCores = Math.min( - roundUp( - Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), - minimumResource.getVirtualCores()), - maximumResource.getVirtualCores()); + Resource maximumResource, Resource stepFactor) { + int normalizedMemory = normalizeInt(r.getMemory(), + minimumResource.getMemory(), maximumResource.getMemory(), + stepFactor.getMemory()); + int normalizedCores = normalizeInt(r.getVirtualCores(), + minimumResource.getVirtualCores(), maximumResource.getVirtualCores(), + stepFactor.getVirtualCores()); return Resources.createResource(normalizedMemory, normalizedCores); } @Override - public Resource roundUp(Resource r, Resource minimumResource) { + public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(), minimumResource.getMemory()), - roundUp(r.getVirtualCores(), minimumResource.getVirtualCores()) + roundUp(r.getMemory(), stepFactor.getMemory()), + roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()) ); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java index b2dd19b..1cb96d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java @@ -51,6 +51,22 @@ public static int roundDown(int a, int b) { return (a / b) * b; } + public static int normalizeInt(int value, int min, int max, int step) { + if (value > min) { + if (value < max) { + value = roundUp(value, step); + if (value > max) { + value = max; + } + } else { + value = max; + } + } else { + value = min; + } + return value; + } + /** * Compute the number of containers which can be allocated given * available and required resources. @@ -97,7 +113,7 @@ public abstract Resource multiplyAndNormalizeDown( * @return normalized resource */ public abstract Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource); + Resource maximumResource, Resource multiplierResource); /** * Round-up resource r given factor stepFactor. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java index 58cd676..8af1efa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java @@ -132,9 +132,9 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) { } public static Resource normalize( - ResourceCalculator calculator, Resource lhs, Resource factor, - Resource limit) { - return calculator.normalize(lhs, factor, limit); + ResourceCalculator calculator, Resource lhs, Resource min, + Resource max, Resource factor) { + return calculator.normalize(lhs, min, max, factor); } public static Resource roundUp( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c0a54c7..cf7b506 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -90,11 +90,12 @@ public static void normalizeRequests( ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, - Resource maximumResource) { + Resource maximumResource, + Resource multiplierResource) { for (ResourceRequest ask : asks) { normalizeRequest( ask, resourceCalculator, clusterResource, minimumResource, - maximumResource); + maximumResource, multiplierResource); } } @@ -107,11 +108,12 @@ public static void normalizeRequest( ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, - Resource maximumResource) { + Resource maximumResource, + Resource multiplierResource) { Resource normalized = Resources.normalize( resourceCalculator, ask.getCapability(), minimumResource, - maximumResource); + maximumResource, multiplierResource); ask.setCapability(normalized); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index f084649..69d0a5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -80,6 +80,14 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, public Resource getMaximumResourceCapability(); /** + * Get multiplier allocatable {@link Resource}. + * @return multiplier allocatable resource + */ + @Public + @Stable + public Resource getMultiplierResourceCapability(); + + /** * Get the number of nodes available in the cluster. * @return the number of available nodes. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index aca2a12..6bf5c68 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,6 +137,7 @@ public Configuration getConf() { private Resource minimumAllocation; private Resource maximumAllocation; + private Resource multiplierAllocation; private Map applications = new ConcurrentHashMap(); @@ -177,6 +178,11 @@ public Resource getMaximumResourceCapability() { } @Override + public Resource getMultiplierResourceCapability() { + return multiplierAllocation; + } + + @Override public Comparator getApplicationComparator() { return applicationComparator; } @@ -214,6 +220,7 @@ public Resource getClusterResources() { this.minimumAllocation = this.conf.getMinimumAllocation(); this.maximumAllocation = this.conf.getMaximumAllocation(); + this.multiplierAllocation = this.conf.getMultiplierAllocation(); this.calculator = this.conf.getResourceCalculator(); this.rmContext = rmContext; @@ -484,7 +491,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, // Sanity check SchedulerUtils.normalizeRequests( ask, calculator, getClusterResources(), minimumAllocation, - maximumAllocation); + maximumAllocation, multiplierAllocation); // Release containers for (ContainerId releasedContainerId : release) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8d71f00..586f812 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -320,6 +320,16 @@ public Resource getMaximumAllocation() { return Resources.createResource(maximumMemory, maximumCores); } + public Resource getMultiplierAllocation() { + int multiplierMem = getInt( + YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB); + int multiplierCores = getInt( + YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES); + return Resources.createResource(multiplierMem, multiplierCores); + } + public boolean getEnableUserMetrics() { return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 5904ae8..85d93e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -113,6 +113,7 @@ private RMContext rmContext; private Resource minimumAllocation; private Resource maximumAllocation; + private Resource multiplierAllocation; private QueueManager queueMgr; private Clock clock; @@ -524,6 +525,11 @@ public Resource getMaximumResourceCapability() { return maximumAllocation; } + @Override + public Resource getMultiplierResourceCapability() { + return multiplierAllocation; + } + public double getNodeLocalityThreshold() { return nodeLocalityThreshold; } @@ -721,11 +727,12 @@ private synchronized void removeNode(RMNode rmNode) { * @param asks a list of resource requests * @param minMemory the configured minimum memory allocation * @param maxMemory the configured maximum memory allocation + * @param multiplierMemory the configured multiplier memory allocation */ static void normalizeRequests(List asks, - int minMemory, int maxMemory) { + int minMemory, int maxMemory, int multiplierMemory) { for (ResourceRequest ask : asks) { - normalizeRequest(ask, minMemory, maxMemory); + normalizeRequest(ask, minMemory, maxMemory, multiplierMemory); } } @@ -736,13 +743,13 @@ static void normalizeRequests(List asks, * @param ask the resource request * @param minMemory the configured minimum memory allocation * @param maxMemory the configured maximum memory allocation + * @param multiplierMemory the configured multiplier memory allocation */ static void normalizeRequest(ResourceRequest ask, int minMemory, - int maxMemory) { - int memory = Math.max(ask.getCapability().getMemory(), minMemory); - int normalizedMemory = - minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0)); - ask.getCapability().setMemory(Math.min(normalizedMemory, maxMemory)); + int maxMemory, int multiplierMemory) { + int normalizedMemory = ResourceCalculator.normalizeInt( + ask.getCapability().getMemory(), minMemory, maxMemory, multiplierMemory); + ask.getCapability().setMemory(normalizedMemory); } @Override @@ -759,7 +766,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // Sanity check normalizeRequests(ask, minimumAllocation.getMemory(), - maximumAllocation.getMemory()); + maximumAllocation.getMemory(), multiplierAllocation.getMemory()); // Release containers for (ContainerId releasedContainerId : release) { @@ -1017,6 +1024,7 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) this.conf = new FairSchedulerConfiguration(conf); minimumAllocation = this.conf.getMinimumMemoryAllocation(); maximumAllocation = this.conf.getMaximumMemoryAllocation(); + multiplierAllocation = this.conf.getMultiplierMemoryAllocation(); userAsDefaultQueue = this.conf.getUserAsDefaultQueue(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 05307cc..257c34f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -92,6 +92,13 @@ public Resource getMaximumMemoryAllocation() { return Resources.createResource(mem); } + public Resource getMultiplierMemoryAllocation() { + int mem = getInt( + YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB); + return Resources.createResource(mem); + } + public boolean getUserAsDefaultQueue() { return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 2024e74..fa2e30c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -110,7 +110,8 @@ private boolean initialized; private Resource minimumAllocation; private Resource maximumAllocation; - + private Resource multiplierAllocation; + private Map applications = new TreeMap(); @@ -197,6 +198,11 @@ public Resource getMaximumResourceCapability() { } @Override + public Resource getMultiplierResourceCapability() { + return multiplierAllocation; + } + + @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { @@ -211,6 +217,10 @@ public Resource getMaximumResourceCapability() { Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + this.multiplierAllocation = + Resources.createResource(conf.getInt( + YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB)); this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); this.activeUsersManager = new ActiveUsersManager(metrics); @@ -233,7 +243,8 @@ public Allocation allocate( // Sanity check SchedulerUtils.normalizeRequests(ask, resourceCalculator, - clusterResource, minimumAllocation, maximumAllocation); + clusterResource, minimumAllocation, maximumAllocation, + multiplierAllocation); // Release containers for (ContainerId releasedContainer : release) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index f17da43..9545d13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -42,45 +42,47 @@ public void testNormalizeRequest() { final int minMemory = 1024; final int maxMemory = 8192; + final int multMemory = 256; Resource minResource = Resources.createResource(minMemory, 0); Resource maxResource = Resources.createResource(maxMemory, 0); + Resource multResource = Resources.createResource(multMemory, 0); ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory ask.setCapability(Resources.createResource(-1024)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(minMemory, ask.getCapability().getMemory()); // case zero memory ask.setCapability(Resources.createResource(0)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(minMemory, ask.getCapability().getMemory()); // case memory is a multiple of minMemory ask.setCapability(Resources.createResource(2 * minMemory)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(2 * minMemory, ask.getCapability().getMemory()); // case memory is not a multiple of minMemory ask.setCapability(Resources.createResource(minMemory + 10)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); - assertEquals(2 * minMemory, ask.getCapability().getMemory()); + maxResource, multResource); + assertEquals(minMemory + multMemory, ask.getCapability().getMemory()); // case memory is equal to max allowed ask.setCapability(Resources.createResource(maxMemory)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(maxMemory, ask.getCapability().getMemory()); // case memory is just less than max ask.setCapability(Resources.createResource(maxMemory - 10)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(maxMemory, ask.getCapability().getMemory()); // max is not a multiple of min @@ -88,14 +90,14 @@ public void testNormalizeRequest() { ask.setCapability(Resources.createResource(maxMemory - 100)); // multiple of minMemory > maxMemory, then reduce to maxMemory SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); // ask is more than max maxResource = Resources.createResource(maxMemory, 0); ask.setCapability(Resources.createResource(maxMemory + 100)); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, - maxResource); + maxResource, multResource); assertEquals(maxResource.getMemory(), ask.getCapability().getMemory()); } @@ -105,6 +107,7 @@ public void testNormalizeRequestWithDominantResourceCalculator() { Resource minResource = Resources.createResource(1024, 1); Resource maxResource = Resources.createResource(10240, 10); + Resource multResource = Resources.createResource(256, 2); Resource clusterResource = Resources.createResource(10 * 1024, 10); ResourceRequest ask = new ResourceRequestPBImpl(); @@ -112,24 +115,27 @@ public void testNormalizeRequestWithDominantResourceCalculator() { // case negative memory/vcores ask.setCapability(Resources.createResource(-1024, -1)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, clusterResource, minResource, maxResource, + multResource); assertEquals(minResource, ask.getCapability()); // case zero memory/vcores ask.setCapability(Resources.createResource(0, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, clusterResource, minResource, maxResource, + multResource); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemory()); // case non-zero memory & zero cores - ask.setCapability(Resources.createResource(1536, 0)); + ask.setCapability(Resources.createResource(1200, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); - assertEquals(Resources.createResource(2048, 1), ask.getCapability()); + ask, resourceCalculator, clusterResource, minResource, maxResource, + multResource); + assertEquals(Resources.createResource(1280, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); - assertEquals(2048, ask.getCapability().getMemory()); + assertEquals(1280, ask.getCapability().getMemory()); } @Test (timeout = 30000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 1bb6a0e..3934c1f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -375,6 +375,28 @@ public void testParseQueue() throws IOException { } @Test + public void testMinMaxMultiplierResourcesSettings() throws IOException { + CapacityScheduler cs = new CapacityScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + cs.setConf(conf); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 10); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES, 3); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(128, cs.getMultiplierResourceCapability().getMemory()); + Assert.assertEquals(10, cs.getMaximumResourceCapability().getVirtualCores()); + Assert.assertEquals(2, cs.getMinimumResourceCapability().getVirtualCores()); + Assert.assertEquals(3, cs.getMultiplierResourceCapability().getVirtualCores()); + } + + @Test public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index f5c0d55..1fcf1ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -208,6 +208,7 @@ public void testLoadConfigurationOnInitialize() throws IOException { conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128); scheduler.reinitialize(conf, resourceManager.getRMContext()); Assert.assertEquals(true, scheduler.assignMultiple); Assert.assertEquals(3, scheduler.maxAssign); @@ -216,6 +217,7 @@ public void testLoadConfigurationOnInitialize() throws IOException { Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01); Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory()); Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(128, scheduler.getMultiplierResourceCapability().getMemory()); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 0ee3a52..f6835c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; import org.junit.Before; @@ -117,6 +119,22 @@ private ResourceRequest createResourceRequest(int memory, String host, return request; } + @Test + public void testMinMaxMultiplierResourcesSettings() throws IOException { + FifoScheduler cs = new FifoScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + cs.setConf(conf); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(128, cs.getMultiplierResourceCapability().getMemory()); + } + @Test(timeout=5000) public void testFifoSchedulerCapacityWhenNoNMs() { FifoScheduler scheduler = new FifoScheduler();