diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
index 332b88f..6ed7cd6 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
@@ -1328,7 +1328,7 @@ private static AppContext createAppContext(
when(context.getJob(isA(JobId.class))).thenReturn(job);
when(context.getClusterInfo()).thenReturn(
new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils
- .newResource(10240, 1)));
+ .newResource(10240, 1), BuilderUtils.newResource(256, 1)));
when(context.getEventHandler()).thenReturn(new EventHandler() {
@Override
public void handle(Event event) {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 25849f4..9c05465 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -199,7 +199,8 @@ public void verifySlotMillis(int mapMemMb, int reduceMemMb,
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
app.setClusterInfo(new ClusterInfo(BuilderUtils
- .newResource(minContainerSize, 1), BuilderUtils.newResource(10240, 1)));
+ .newResource(minContainerSize, 1), BuilderUtils.newResource(10240, 1),
+ BuilderUtils.newResource(256, 1)));
Job job = app.submit(conf);
app.waitForState(job, JobState.RUNNING);
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index 6db4816..9103868 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -118,7 +118,7 @@ private static AppContext createAppContext() {
when(ctx.getJob(isA(JobId.class))).thenReturn(job);
when(ctx.getClusterInfo()).thenReturn(
new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils
- .newResource(10240, 1)));
+ .newResource(10240, 1), BuilderUtils.newResource(256, 1)));
when(ctx.getEventHandler()).thenReturn(eventHandler);
return ctx;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 8b1b709..8a66718 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -72,6 +72,19 @@
public void setMaximumResourceCapability(Resource capability);
/**
+ * Get the multiplier capability for any {@link Resource} allocated by the
+ * ResourceManager in the cluster.
+ * @return multiplier capability of allocated resources in the cluster
+ */
+ @Public
+ @Stable
+ public Resource getMultiplierResourceCapability();
+
+ @Private
+ @Unstable
+ public void setMultiplierResourceCapability(Resource capability);
+
+ /**
* Get the ApplicationACLs for the application.
* @return all the ApplicationACLs
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index b861282..5037fa9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -46,6 +46,7 @@
private Resource minimumResourceCapability;
private Resource maximumResourceCapability;
+ private Resource multiplierResourceCapability;
private Map applicationACLS = null;
public RegisterApplicationMasterResponsePBImpl() {
@@ -79,7 +80,11 @@ private void mergeLocalToBuilder() {
}
if (this.maximumResourceCapability != null) {
builder.setMaximumCapability(
- convertToProtoFormat(this.maximumResourceCapability));
+ convertToProtoFormat(this.maximumResourceCapability));
+ }
+ if (this.multiplierResourceCapability != null) {
+ builder.setMultiplierCapability(
+ convertToProtoFormat(this.multiplierResourceCapability));
}
if (this.applicationACLS != null) {
addApplicationACLs();
@@ -125,6 +130,22 @@ public Resource getMinimumResourceCapability() {
}
@Override
+ public Resource getMultiplierResourceCapability() {
+ if (this.multiplierResourceCapability != null) {
+ return this.multiplierResourceCapability;
+ }
+
+ RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasMultiplierCapability()) {
+ return null;
+ }
+
+ this.multiplierResourceCapability
+ = convertFromProtoFormat(p.getMultiplierCapability());
+ return this.multiplierResourceCapability;
+ }
+
+ @Override
public void setMaximumResourceCapability(Resource capability) {
maybeInitBuilder();
if(maximumResourceCapability == null) {
@@ -142,6 +163,14 @@ public void setMinimumResourceCapability(Resource capability) {
this.minimumResourceCapability = capability;
}
+ @Override
+ public void setMultiplierResourceCapability(Resource capability) {
+ maybeInitBuilder();
+ if(multiplierResourceCapability == null) {
+ builder.clearMultiplierCapability();
+ }
+ this.multiplierResourceCapability = capability;
+ }
@Override
public Map getApplicationACLs() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 6ac0274..2db04f1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -38,6 +38,7 @@ message RegisterApplicationMasterResponseProto {
optional ResourceProto minimumCapability = 1;
optional ResourceProto maximumCapability = 2;
repeated ApplicationACLMapProto application_ACLs = 3;
+ optional ResourceProto multiplierCapability = 4;
}
message FinishApplicationMasterRequestProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
index 2764061..28333a6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -33,6 +34,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -143,8 +145,11 @@ public void testAMRMClient() throws YarnRemoteException, IOException {
amClient.init(conf);
amClient.start();
+ RegisterApplicationMasterResponse response =
amClient.registerApplicationMaster("Host", 10000, "");
+ testMinMaxMultiplerResourceValues(response);
+
testAllocation(amClient);
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
@@ -156,7 +161,27 @@ public void testAMRMClient() throws YarnRemoteException, IOException {
}
}
}
-
+
+ private void testMinMaxMultiplerResourceValues(
+ RegisterApplicationMasterResponse response)
+ throws YarnRemoteException, IOException {
+ int minMem = response.getMinimumResourceCapability().getMemory();
+ int maxMem = response.getMaximumResourceCapability().getMemory();
+ int multMem = response.getMultiplierResourceCapability().getMemory();
+ int minCPUs = response.getMinimumResourceCapability().getVirtualCores();
+ int maxCPUs = response.getMaximumResourceCapability().getVirtualCores();
+ int multCPUs = response.getMultiplierResourceCapability().getVirtualCores();
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB,
+ multMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ minCPUs);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ maxCPUs);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES,
+ multCPUs);
+ }
private void testAllocation(final AMRMClientImpl amClient)
throws YarnRemoteException, IOException {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
index 64c7ca7..9cc0c81 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
@@ -24,15 +24,20 @@
public class ClusterInfo {
private Resource minContainerCapability;
private Resource maxContainerCapability;
+ private Resource multiplierContainerCapacity;
public ClusterInfo() {
this.minContainerCapability = Records.newRecord(Resource.class);
this.maxContainerCapability = Records.newRecord(Resource.class);
+ this.multiplierContainerCapacity = Records.newRecord(Resource.class);
}
- public ClusterInfo(Resource minCapability, Resource maxCapability) {
+ public ClusterInfo(Resource minCapability, Resource maxCapability,
+ Resource multiplierContainerCapacity) {
this.minContainerCapability = minCapability;
this.maxContainerCapability = maxCapability;
+ this.multiplierContainerCapacity = multiplierContainerCapacity;
+
}
public Resource getMinContainerCapability() {
@@ -50,4 +55,14 @@ public Resource getMaxContainerCapability() {
public void setMaxContainerCapability(Resource maxContainerCapability) {
this.maxContainerCapability = maxContainerCapability;
}
+
+ public Resource getMultiplierContainerCapability() {
+ return multiplierContainerCapacity;
+ }
+
+ public void setMultiplierContainerCapability(
+ Resource multiplierContainerCapacity) {
+ this.multiplierContainerCapacity = multiplierContainerCapacity;
+ }
+
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f9b017d..58d09e4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -126,6 +126,14 @@
YARN_PREFIX + "scheduler.maximum-allocation-vcores";
public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4;
+ /** Multiplier request grant-able by the RM scheduler. */
+ public static final String RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB =
+ YARN_PREFIX + "scheduler.multiplier-allocation-mb";
+ public static final int DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB = 256;
+ public static final String RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES =
+ YARN_PREFIX + "scheduler.multiplier-allocation-vcores";
+ public static final int DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES = 1;
+
/** Number of threads to handle scheduler interface.*/
public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
RM_PREFIX + "scheduler.client.thread-count";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 599f8a9..49f1c69 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -228,7 +228,7 @@
in terms of virtual CPU cores. Requests higher than this won't take effect,
and will get capped to this value.
yarn.scheduler.maximum-allocation-vcores
- 32
+ 4
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 3094a93..c8e1e0a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -211,6 +211,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
.getMinimumResourceCapability());
response.setMaximumResourceCapability(rScheduler
.getMaximumResourceCapability());
+ response.setMultiplierResourceCapability(rScheduler
+ .getMultiplierResourceCapability());
response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)
.getSubmissionContext().getAMContainerSpec().getApplicationACLs());
return response;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
index 5a691af..65191f9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
@@ -54,19 +54,17 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- int normalizedMemory = Math.min(
- roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
- minimumResource.getMemory()),
- maximumResource.getMemory());
+ Resource maximumResource, Resource stepFactor) {
+ int normalizedMemory = normalizeInt(r.getMemory(),
+ minimumResource.getMemory(), maximumResource.getMemory(),
+ stepFactor.getMemory());
return Resources.createResource(normalizedMemory);
}
@Override
- public Resource roundUp(Resource r, Resource minimumResource) {
+ public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemory(),minimumResource.getMemory())
+ roundUp(r.getMemory(), stepFactor.getMemory())
);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
index 2f66990..ddf8f00 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
@@ -124,26 +124,22 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- int normalizedMemory = Math.min(
- roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
- minimumResource.getMemory()),
- maximumResource.getMemory());
- int normalizedCores = Math.min(
- roundUp(
- Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()),
- minimumResource.getVirtualCores()),
- maximumResource.getVirtualCores());
+ Resource maximumResource, Resource stepFactor) {
+ int normalizedMemory = normalizeInt(r.getMemory(),
+ minimumResource.getMemory(), maximumResource.getMemory(),
+ stepFactor.getMemory());
+ int normalizedCores = normalizeInt(r.getVirtualCores(),
+ minimumResource.getVirtualCores(), maximumResource.getVirtualCores(),
+ stepFactor.getVirtualCores());
return Resources.createResource(normalizedMemory,
normalizedCores);
}
@Override
- public Resource roundUp(Resource r, Resource minimumResource) {
+ public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemory(), minimumResource.getMemory()),
- roundUp(r.getVirtualCores(), minimumResource.getVirtualCores())
+ roundUp(r.getMemory(), stepFactor.getMemory()),
+ roundUp(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
index b2dd19b..1cb96d7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
@@ -51,6 +51,22 @@ public static int roundDown(int a, int b) {
return (a / b) * b;
}
+ public static int normalizeInt(int value, int min, int max, int step) {
+ if (value > min) {
+ if (value < max) {
+ value = roundUp(value, step);
+ if (value > max) {
+ value = max;
+ }
+ } else {
+ value = max;
+ }
+ } else {
+ value = min;
+ }
+ return value;
+ }
+
/**
* Compute the number of containers which can be allocated given
* available and required resources.
@@ -97,7 +113,7 @@ public abstract Resource multiplyAndNormalizeDown(
* @return normalized resource
*/
public abstract Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource);
+ Resource maximumResource, Resource multiplierResource);
/**
* Round-up resource r given factor stepFactor.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
index 58cd676..8af1efa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
@@ -132,9 +132,9 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) {
}
public static Resource normalize(
- ResourceCalculator calculator, Resource lhs, Resource factor,
- Resource limit) {
- return calculator.normalize(lhs, factor, limit);
+ ResourceCalculator calculator, Resource lhs, Resource min,
+ Resource max, Resource factor) {
+ return calculator.normalize(lhs, min, max, factor);
}
public static Resource roundUp(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c0a54c7..cf7b506 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -90,11 +90,12 @@ public static void normalizeRequests(
ResourceCalculator resourceCalculator,
Resource clusterResource,
Resource minimumResource,
- Resource maximumResource) {
+ Resource maximumResource,
+ Resource multiplierResource) {
for (ResourceRequest ask : asks) {
normalizeRequest(
ask, resourceCalculator, clusterResource, minimumResource,
- maximumResource);
+ maximumResource, multiplierResource);
}
}
@@ -107,11 +108,12 @@ public static void normalizeRequest(
ResourceCalculator resourceCalculator,
Resource clusterResource,
Resource minimumResource,
- Resource maximumResource) {
+ Resource maximumResource,
+ Resource multiplierResource) {
Resource normalized =
Resources.normalize(
resourceCalculator, ask.getCapability(), minimumResource,
- maximumResource);
+ maximumResource, multiplierResource);
ask.setCapability(normalized);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index f084649..69d0a5c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -80,6 +80,14 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
public Resource getMaximumResourceCapability();
/**
+ * Get multiplier allocatable {@link Resource}.
+ * @return multiplier allocatable resource
+ */
+ @Public
+ @Stable
+ public Resource getMultiplierResourceCapability();
+
+ /**
* Get the number of nodes available in the cluster.
* @return the number of available nodes.
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aca2a12..6bf5c68 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,6 +137,7 @@ public Configuration getConf() {
private Resource minimumAllocation;
private Resource maximumAllocation;
+ private Resource multiplierAllocation;
private Map applications =
new ConcurrentHashMap();
@@ -177,6 +178,11 @@ public Resource getMaximumResourceCapability() {
}
@Override
+ public Resource getMultiplierResourceCapability() {
+ return multiplierAllocation;
+ }
+
+ @Override
public Comparator getApplicationComparator() {
return applicationComparator;
}
@@ -214,6 +220,7 @@ public Resource getClusterResources() {
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
+ this.multiplierAllocation = this.conf.getMultiplierAllocation();
this.calculator = this.conf.getResourceCalculator();
this.rmContext = rmContext;
@@ -484,7 +491,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
// Sanity check
SchedulerUtils.normalizeRequests(
ask, calculator, getClusterResources(), minimumAllocation,
- maximumAllocation);
+ maximumAllocation, multiplierAllocation);
// Release containers
for (ContainerId releasedContainerId : release) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 8d71f00..586f812 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -320,6 +320,16 @@ public Resource getMaximumAllocation() {
return Resources.createResource(maximumMemory, maximumCores);
}
+ public Resource getMultiplierAllocation() {
+ int multiplierMem = getInt(
+ YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB);
+ int multiplierCores = getInt(
+ YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES);
+ return Resources.createResource(multiplierMem, multiplierCores);
+ }
+
public boolean getEnableUserMetrics() {
return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 5904ae8..85d93e5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -113,6 +113,7 @@
private RMContext rmContext;
private Resource minimumAllocation;
private Resource maximumAllocation;
+ private Resource multiplierAllocation;
private QueueManager queueMgr;
private Clock clock;
@@ -524,6 +525,11 @@ public Resource getMaximumResourceCapability() {
return maximumAllocation;
}
+ @Override
+ public Resource getMultiplierResourceCapability() {
+ return multiplierAllocation;
+ }
+
public double getNodeLocalityThreshold() {
return nodeLocalityThreshold;
}
@@ -721,11 +727,12 @@ private synchronized void removeNode(RMNode rmNode) {
* @param asks a list of resource requests
* @param minMemory the configured minimum memory allocation
* @param maxMemory the configured maximum memory allocation
+ * @param multiplierMemory the configured multiplier memory allocation
*/
static void normalizeRequests(List asks,
- int minMemory, int maxMemory) {
+ int minMemory, int maxMemory, int multiplierMemory) {
for (ResourceRequest ask : asks) {
- normalizeRequest(ask, minMemory, maxMemory);
+ normalizeRequest(ask, minMemory, maxMemory, multiplierMemory);
}
}
@@ -736,13 +743,13 @@ static void normalizeRequests(List asks,
* @param ask the resource request
* @param minMemory the configured minimum memory allocation
* @param maxMemory the configured maximum memory allocation
+ * @param multiplierMemory the configured multiplier memory allocation
*/
static void normalizeRequest(ResourceRequest ask, int minMemory,
- int maxMemory) {
- int memory = Math.max(ask.getCapability().getMemory(), minMemory);
- int normalizedMemory =
- minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0));
- ask.getCapability().setMemory(Math.min(normalizedMemory, maxMemory));
+ int maxMemory, int multiplierMemory) {
+ int normalizedMemory = ResourceCalculator.normalizeInt(
+ ask.getCapability().getMemory(), minMemory, maxMemory, multiplierMemory);
+ ask.getCapability().setMemory(normalizedMemory);
}
@Override
@@ -759,7 +766,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId,
// Sanity check
normalizeRequests(ask, minimumAllocation.getMemory(),
- maximumAllocation.getMemory());
+ maximumAllocation.getMemory(), multiplierAllocation.getMemory());
// Release containers
for (ContainerId releasedContainerId : release) {
@@ -1017,6 +1024,7 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext)
this.conf = new FairSchedulerConfiguration(conf);
minimumAllocation = this.conf.getMinimumMemoryAllocation();
maximumAllocation = this.conf.getMaximumMemoryAllocation();
+ multiplierAllocation = this.conf.getMultiplierMemoryAllocation();
userAsDefaultQueue = this.conf.getUserAsDefaultQueue();
nodeLocalityThreshold = this.conf.getLocalityThresholdNode();
rackLocalityThreshold = this.conf.getLocalityThresholdRack();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 05307cc..257c34f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -92,6 +92,13 @@ public Resource getMaximumMemoryAllocation() {
return Resources.createResource(mem);
}
+ public Resource getMultiplierMemoryAllocation() {
+ int mem = getInt(
+ YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB);
+ return Resources.createResource(mem);
+ }
+
public boolean getUserAsDefaultQueue() {
return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 2024e74..fa2e30c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -110,7 +110,8 @@
private boolean initialized;
private Resource minimumAllocation;
private Resource maximumAllocation;
-
+ private Resource multiplierAllocation;
+
private Map applications
= new TreeMap();
@@ -197,6 +198,11 @@ public Resource getMaximumResourceCapability() {
}
@Override
+ public Resource getMultiplierResourceCapability() {
+ return multiplierAllocation;
+ }
+
+ @Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException
{
@@ -211,6 +217,10 @@ public Resource getMaximumResourceCapability() {
Resources.createResource(conf.getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+ this.multiplierAllocation =
+ Resources.createResource(conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB));
this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
conf);
this.activeUsersManager = new ActiveUsersManager(metrics);
@@ -233,7 +243,8 @@ public Allocation allocate(
// Sanity check
SchedulerUtils.normalizeRequests(ask, resourceCalculator,
- clusterResource, minimumAllocation, maximumAllocation);
+ clusterResource, minimumAllocation, maximumAllocation,
+ multiplierAllocation);
// Release containers
for (ContainerId releasedContainer : release) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index f17da43..9545d13 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -42,45 +42,47 @@ public void testNormalizeRequest() {
final int minMemory = 1024;
final int maxMemory = 8192;
+ final int multMemory = 256;
Resource minResource = Resources.createResource(minMemory, 0);
Resource maxResource = Resources.createResource(maxMemory, 0);
+ Resource multResource = Resources.createResource(multMemory, 0);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory
ask.setCapability(Resources.createResource(-1024));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(minMemory, ask.getCapability().getMemory());
// case zero memory
ask.setCapability(Resources.createResource(0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(minMemory, ask.getCapability().getMemory());
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
- assertEquals(2 * minMemory, ask.getCapability().getMemory());
+ maxResource, multResource);
+ assertEquals(minMemory + multMemory, ask.getCapability().getMemory());
// case memory is equal to max allowed
ask.setCapability(Resources.createResource(maxMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(maxMemory, ask.getCapability().getMemory());
// case memory is just less than max
ask.setCapability(Resources.createResource(maxMemory - 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(maxMemory, ask.getCapability().getMemory());
// max is not a multiple of min
@@ -88,14 +90,14 @@ public void testNormalizeRequest() {
ask.setCapability(Resources.createResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, multResource);
assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}
@@ -105,6 +107,7 @@ public void testNormalizeRequestWithDominantResourceCalculator() {
Resource minResource = Resources.createResource(1024, 1);
Resource maxResource = Resources.createResource(10240, 10);
+ Resource multResource = Resources.createResource(256, 2);
Resource clusterResource = Resources.createResource(10 * 1024, 10);
ResourceRequest ask = new ResourceRequestPBImpl();
@@ -112,24 +115,27 @@ public void testNormalizeRequestWithDominantResourceCalculator() {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ multResource);
assertEquals(minResource, ask.getCapability());
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ multResource);
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemory());
// case non-zero memory & zero cores
- ask.setCapability(Resources.createResource(1536, 0));
+ ask.setCapability(Resources.createResource(1200, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
- assertEquals(Resources.createResource(2048, 1), ask.getCapability());
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ multResource);
+ assertEquals(Resources.createResource(1280, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
- assertEquals(2048, ask.getCapability().getMemory());
+ assertEquals(1280, ask.getCapability().getMemory());
}
@Test (timeout = 30000)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 1bb6a0e..3934c1f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -375,6 +375,28 @@ public void testParseQueue() throws IOException {
}
@Test
+ public void testMinMaxMultiplierResourcesSettings() throws IOException {
+ CapacityScheduler cs = new CapacityScheduler();
+ YarnConfiguration conf = new YarnConfiguration();
+ cs.setConf(conf);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 10);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_VCORES, 3);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory());
+ Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, cs.getMultiplierResourceCapability().getMemory());
+ Assert.assertEquals(10, cs.getMaximumResourceCapability().getVirtualCores());
+ Assert.assertEquals(2, cs.getMinimumResourceCapability().getVirtualCores());
+ Assert.assertEquals(3, cs.getMultiplierResourceCapability().getVirtualCores());
+ }
+
+ @Test
public void testReconnectedNode() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index f5c0d55..1fcf1ea 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -208,6 +208,7 @@ public void testLoadConfigurationOnInitialize() throws IOException {
conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128);
scheduler.reinitialize(conf, resourceManager.getRMContext());
Assert.assertEquals(true, scheduler.assignMultiple);
Assert.assertEquals(3, scheduler.maxAssign);
@@ -216,6 +217,7 @@ public void testLoadConfigurationOnInitialize() throws IOException {
Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01);
Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory());
Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, scheduler.getMultiplierResourceCapability().getMemory());
}
@Test
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 0ee3a52..f6835c5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -56,6 +56,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
import org.junit.Before;
@@ -117,6 +119,22 @@ private ResourceRequest createResourceRequest(int memory, String host,
return request;
}
+ @Test
+ public void testMinMaxMultiplierResourcesSettings() throws IOException {
+ FifoScheduler cs = new FifoScheduler();
+ YarnConfiguration conf = new YarnConfiguration();
+ cs.setConf(conf);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MULTIPLIER_ALLOCATION_MB, 128);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory());
+ Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, cs.getMultiplierResourceCapability().getMemory());
+ }
+
@Test(timeout=5000)
public void testFifoSchedulerCapacityWhenNoNMs() {
FifoScheduler scheduler = new FifoScheduler();