diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
index 41e88a8..1a88181 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
@@ -1328,7 +1328,7 @@ private static AppContext createAppContext(
when(context.getJob(isA(JobId.class))).thenReturn(job);
when(context.getClusterInfo()).thenReturn(
new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance(
- 10240, 1)));
+ 10240, 1), BuilderUtils.newResource(256, 1)));
when(context.getEventHandler()).thenReturn(new EventHandler() {
@Override
public void handle(Event event) {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 1516849..6710a2d 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -199,7 +199,8 @@ public void verifySlotMillis(int mapMemMb, int reduceMemMb,
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
app.setClusterInfo(new ClusterInfo(Resource
- .newInstance(minContainerSize, 1), Resource.newInstance(10240, 1)));
+ .newInstance(minContainerSize, 1), Resource.newInstance(10240, 1),
+ BuilderUtils.newResource(256, 1)));
Job job = app.submit(conf);
app.waitForState(job, JobState.RUNNING);
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index a0ed206..9abc359 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -119,7 +119,7 @@ private static AppContext createAppContext() {
when(ctx.getJob(isA(JobId.class))).thenReturn(job);
when(ctx.getClusterInfo()).thenReturn(
new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance(
- 10240, 1)));
+ 10240, 1), BuilderUtils.newResource(256, 1)));
when(ctx.getEventHandler()).thenReturn(eventHandler);
return ctx;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 8b1b709..00109ca 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -36,6 +36,7 @@
*
* - Minimum capability for allocated resources in the cluster.
* - Maximum capability for allocated resources in the cluster.
+ * - Increment capability for allocated resources in the cluster.
*
*
*
@@ -72,6 +73,19 @@
public void setMaximumResourceCapability(Resource capability);
/**
+ * Get the increment capability for any {@link Resource} allocated by the
+ * ResourceManager in the cluster.
+ * @return incremente capability of allocated resources in the cluster
+ */
+ @Public
+ @Stable
+ public Resource getIncrementResourceCapability();
+
+ @Private
+ @Unstable
+ public void setIncrementResourceCapability(Resource capability);
+
+ /**
* Get the ApplicationACLs for the application.
* @return all the ApplicationACLs
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index b861282..f0f4acd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -46,6 +46,7 @@
private Resource minimumResourceCapability;
private Resource maximumResourceCapability;
+ private Resource incrementResourceCapability;
private Map applicationACLS = null;
public RegisterApplicationMasterResponsePBImpl() {
@@ -79,7 +80,11 @@ private void mergeLocalToBuilder() {
}
if (this.maximumResourceCapability != null) {
builder.setMaximumCapability(
- convertToProtoFormat(this.maximumResourceCapability));
+ convertToProtoFormat(this.maximumResourceCapability));
+ }
+ if (this.incrementResourceCapability != null) {
+ builder.setIncrementCapability(
+ convertToProtoFormat(this.incrementResourceCapability));
}
if (this.applicationACLS != null) {
addApplicationACLs();
@@ -125,6 +130,22 @@ public Resource getMinimumResourceCapability() {
}
@Override
+ public Resource getIncrementResourceCapability() {
+ if (this.incrementResourceCapability != null) {
+ return this.incrementResourceCapability;
+ }
+
+ RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasIncrementCapability()) {
+ return null;
+ }
+
+ this.incrementResourceCapability
+ = convertFromProtoFormat(p.getIncrementCapability());
+ return this.incrementResourceCapability;
+ }
+
+ @Override
public void setMaximumResourceCapability(Resource capability) {
maybeInitBuilder();
if(maximumResourceCapability == null) {
@@ -142,6 +163,14 @@ public void setMinimumResourceCapability(Resource capability) {
this.minimumResourceCapability = capability;
}
+ @Override
+ public void setIncrementResourceCapability(Resource capability) {
+ maybeInitBuilder();
+ if(incrementResourceCapability == null) {
+ builder.clearIncrementCapability();
+ }
+ this.incrementResourceCapability = capability;
+ }
@Override
public Map getApplicationACLs() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fa1e740..6f9b2b1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -123,6 +123,12 @@
YARN_PREFIX + "scheduler.maximum-allocation-vcores";
public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4;
+ /** Increment request grant-able by the RM scheduler. */
+ public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB =
+ YARN_PREFIX + "scheduler.increment-allocation-mb";
+ public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES =
+ YARN_PREFIX + "scheduler.increment-allocation-vcores";
+
/** Number of threads to handle scheduler interface.*/
public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
RM_PREFIX + "scheduler.client.thread-count";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index ed3f871..7bf9a3f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -38,6 +38,7 @@ message RegisterApplicationMasterResponseProto {
optional ResourceProto minimumCapability = 1;
optional ResourceProto maximumCapability = 2;
repeated ApplicationACLMapProto application_ACLs = 3;
+ optional ResourceProto incrementCapability = 4;
}
message FinishApplicationMasterRequestProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
index 74a1d16..ef844c5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,6 +35,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -392,8 +394,11 @@ public void testAMRMClient() throws YarnRemoteException, IOException {
amClient.init(conf);
amClient.start();
+ RegisterApplicationMasterResponse response =
amClient.registerApplicationMaster("Host", 10000, "");
+ testMinMaxIncrementResourceValues(response);
+
testAllocation(amClient);
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
@@ -406,6 +411,27 @@ public void testAMRMClient() throws YarnRemoteException, IOException {
}
}
+ private void testMinMaxIncrementResourceValues(
+ RegisterApplicationMasterResponse response)
+ throws YarnRemoteException, IOException {
+ int minMem = response.getMinimumResourceCapability().getMemory();
+ int maxMem = response.getMaximumResourceCapability().getMemory();
+ int incrMem = response.getIncrementResourceCapability().getMemory();
+ int minCPUs = response.getMinimumResourceCapability().getVirtualCores();
+ int maxCPUs = response.getMaximumResourceCapability().getVirtualCores();
+ int incrCPUs = response.getIncrementResourceCapability().getVirtualCores();
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ incrMem);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ minCPUs);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ maxCPUs);
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ incrCPUs);
+ }
+
private void testAllocation(final AMRMClientImpl amClient)
throws YarnRemoteException, IOException {
// setup container request
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
index 64c7ca7..38bfab6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ClusterInfo.java
@@ -24,15 +24,20 @@
public class ClusterInfo {
private Resource minContainerCapability;
private Resource maxContainerCapability;
+ private Resource incrContainerCapacity;
public ClusterInfo() {
this.minContainerCapability = Records.newRecord(Resource.class);
this.maxContainerCapability = Records.newRecord(Resource.class);
+ this.incrContainerCapacity = Records.newRecord(Resource.class);
}
- public ClusterInfo(Resource minCapability, Resource maxCapability) {
+ public ClusterInfo(Resource minCapability, Resource maxCapability,
+ Resource incrementCapability) {
this.minContainerCapability = minCapability;
this.maxContainerCapability = maxCapability;
+ this.incrContainerCapacity = incrementCapability;
+
}
public Resource getMinContainerCapability() {
@@ -50,4 +55,13 @@ public Resource getMaxContainerCapability() {
public void setMaxContainerCapability(Resource maxContainerCapability) {
this.maxContainerCapability = maxContainerCapability;
}
+
+ public Resource getIncrContainerCapability() {
+ return incrContainerCapacity;
+ }
+
+ public void setIncrContainerCapability(Resource incrContainerCapacity) {
+ this.incrContainerCapacity = incrContainerCapacity;
+ }
+
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d71d193..e584b44 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -228,7 +228,7 @@
in terms of virtual CPU cores. Requests higher than this won't take effect,
and will get capped to this value.
yarn.scheduler.maximum-allocation-vcores
- 32
+ 4
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 3094a93..0aa8e72 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -211,6 +211,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
.getMinimumResourceCapability());
response.setMaximumResourceCapability(rScheduler
.getMaximumResourceCapability());
+ response.setIncrementResourceCapability(rScheduler
+ .getIncrementResourceCapability());
response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)
.getSubmissionContext().getAMContainerSpec().getApplicationACLs());
return response;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
index 5a691af..57d82dd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
@@ -54,19 +54,17 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- int normalizedMemory = Math.min(
- roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
- minimumResource.getMemory()),
- maximumResource.getMemory());
+ Resource maximumResource, Resource stepFactor) {
+ int normalizedMemory = normalizeInt(r.getMemory(),
+ minimumResource.getMemory(), maximumResource.getMemory(),
+ stepFactor.getMemory());
return Resources.createResource(normalizedMemory);
}
@Override
- public Resource roundUp(Resource r, Resource minimumResource) {
+ public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemory(),minimumResource.getMemory())
+ roundUp(r.getMemory(), stepFactor.getMemory())
);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
index 2f66990..eaa7370 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
@@ -124,26 +124,22 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- int normalizedMemory = Math.min(
- roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
- minimumResource.getMemory()),
- maximumResource.getMemory());
- int normalizedCores = Math.min(
- roundUp(
- Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()),
- minimumResource.getVirtualCores()),
- maximumResource.getVirtualCores());
+ Resource maximumResource, Resource stepFactor) {
+ int normalizedMemory = normalizeInt(r.getMemory(),
+ minimumResource.getMemory(), maximumResource.getMemory(),
+ stepFactor.getMemory());
+ int normalizedCores = normalizeInt(r.getVirtualCores(),
+ minimumResource.getVirtualCores(), maximumResource.getVirtualCores(),
+ stepFactor.getVirtualCores());
return Resources.createResource(normalizedMemory,
normalizedCores);
}
@Override
- public Resource roundUp(Resource r, Resource minimumResource) {
+ public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
- roundUp(r.getMemory(), minimumResource.getMemory()),
- roundUp(r.getVirtualCores(), minimumResource.getVirtualCores())
+ roundUp(r.getMemory(), stepFactor.getMemory()),
+ roundUp(r.getVirtualCores(), stepFactor.getVirtualCores())
);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
index b2dd19b..ad4fead 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
@@ -51,6 +51,22 @@ public static int roundDown(int a, int b) {
return (a / b) * b;
}
+ public static int normalizeInt(int value, int min, int max, int increment) {
+ if (value > min) {
+ if (value < max) {
+ value = roundUp(value, increment);
+ if (value > max) {
+ value = max;
+ }
+ } else {
+ value = max;
+ }
+ } else {
+ value = min;
+ }
+ return value;
+ }
+
/**
* Compute the number of containers which can be allocated given
* available and required resources.
@@ -94,10 +110,11 @@ public abstract Resource multiplyAndNormalizeDown(
* @param r resource
* @param minimumResource step-factor
* @param maximumResource the upper bound of the resource to be allocated
+ * @param stepFactor the increment for resources to be allocated
* @return normalized resource
*/
public abstract Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource);
+ Resource maximumResource, Resource stepFactor);
/**
* Round-up resource r given factor stepFactor.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
index 58cd676..5e67098 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
@@ -132,9 +132,9 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) {
}
public static Resource normalize(
- ResourceCalculator calculator, Resource lhs, Resource factor,
- Resource limit) {
- return calculator.normalize(lhs, factor, limit);
+ ResourceCalculator calculator, Resource lhs, Resource min,
+ Resource max, Resource increment) {
+ return calculator.normalize(lhs, min, max, increment);
}
public static Resource roundUp(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c0a54c7..0426761 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -90,11 +90,12 @@ public static void normalizeRequests(
ResourceCalculator resourceCalculator,
Resource clusterResource,
Resource minimumResource,
- Resource maximumResource) {
+ Resource maximumResource,
+ Resource incrementResource) {
for (ResourceRequest ask : asks) {
normalizeRequest(
ask, resourceCalculator, clusterResource, minimumResource,
- maximumResource);
+ maximumResource, incrementResource);
}
}
@@ -107,11 +108,12 @@ public static void normalizeRequest(
ResourceCalculator resourceCalculator,
Resource clusterResource,
Resource minimumResource,
- Resource maximumResource) {
+ Resource maximumResource,
+ Resource incrementResource) {
Resource normalized =
Resources.normalize(
resourceCalculator, ask.getCapability(), minimumResource,
- maximumResource);
+ maximumResource, incrementResource);
ask.setCapability(normalized);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index f084649..cfa9631 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -80,6 +80,14 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
public Resource getMaximumResourceCapability();
/**
+ * Get increment allocatable {@link Resource}.
+ * @return increment allocatable resource
+ */
+ @Public
+ @Stable
+ public Resource getIncrementResourceCapability();
+
+ /**
* Get the number of nodes available in the cluster.
* @return the number of available nodes.
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aca2a12..0e8d3ba 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,6 +137,7 @@ public Configuration getConf() {
private Resource minimumAllocation;
private Resource maximumAllocation;
+ private Resource incrementAllocation;
private Map applications =
new ConcurrentHashMap();
@@ -177,6 +178,11 @@ public Resource getMaximumResourceCapability() {
}
@Override
+ public Resource getIncrementResourceCapability() {
+ return incrementAllocation;
+ }
+
+ @Override
public Comparator getApplicationComparator() {
return applicationComparator;
}
@@ -214,6 +220,7 @@ public Resource getClusterResources() {
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
+ this.incrementAllocation = this.conf.getIncrementAllocation();
this.calculator = this.conf.getResourceCalculator();
this.rmContext = rmContext;
@@ -224,6 +231,7 @@ public Resource getClusterResources() {
LOG.info("Initialized CapacityScheduler with " +
"calculator=" + getResourceCalculator().getClass() + ", " +
"minimumAllocation=<" + getMinimumResourceCapability() + ">, " +
+ "incrementAllocation=<" + getIncrementResourceCapability() + ">" +
"maximumAllocation=<" + getMaximumResourceCapability() + ">");
} else {
@@ -484,7 +492,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
// Sanity check
SchedulerUtils.normalizeRequests(
ask, calculator, getClusterResources(), minimumAllocation,
- maximumAllocation);
+ maximumAllocation, incrementAllocation);
// Release containers
for (ContainerId releasedContainerId : release) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 8d71f00..f95ddcd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -320,6 +320,17 @@ public Resource getMaximumAllocation() {
return Resources.createResource(maximumMemory, maximumCores);
}
+ public Resource getIncrementAllocation() {
+ Resource minimum = getMinimumAllocation();
+ int incrementMemory = getInt(
+ YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ minimum.getMemory());
+ int incrementCores = getInt(
+ YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES,
+ minimum.getVirtualCores());
+ return Resources.createResource(incrementMemory, incrementCores);
+ }
+
public boolean getEnableUserMetrics() {
return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index f8cff0e..2121303 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -115,6 +115,7 @@
private RMContext rmContext;
private Resource minimumAllocation;
private Resource maximumAllocation;
+ private Resource incrAllocation;
private QueueManager queueMgr;
private Clock clock;
@@ -261,7 +262,7 @@ private void updatePreemptionVariables() {
*/
boolean isStarvedForMinShare(FSLeafQueue sched) {
Resource desiredShare = Resources.min(RESOURCE_CALCULATOR, clusterCapacity,
- sched.getMinShare(), sched.getDemand());
+ sched.getMinShare(), sched.getDemand());
return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity,
sched.getResourceUsage(), desiredShare);
}
@@ -526,6 +527,11 @@ public Resource getMaximumResourceCapability() {
return maximumAllocation;
}
+ @Override
+ public Resource getIncrementResourceCapability() {
+ return incrAllocation;
+ }
+
public double getNodeLocalityThreshold() {
return nodeLocalityThreshold;
}
@@ -730,7 +736,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId,
// Sanity check
SchedulerUtils.normalizeRequests(ask, new DominantResourceCalculator(),
- clusterCapacity, minimumAllocation, maximumAllocation);
+ clusterCapacity, minimumAllocation, maximumAllocation, incrAllocation);
// Release containers
for (ContainerId releasedContainerId : release) {
@@ -988,6 +994,7 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext)
this.conf = new FairSchedulerConfiguration(conf);
minimumAllocation = this.conf.getMinimumAllocation();
maximumAllocation = this.conf.getMaximumAllocation();
+ incrAllocation = this.conf.getIncrementAllocation();
userAsDefaultQueue = this.conf.getUserAsDefaultQueue();
nodeLocalityThreshold = this.conf.getLocalityThresholdNode();
rackLocalityThreshold = this.conf.getLocalityThresholdRack();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 5696746..39b35b5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -102,6 +102,17 @@ public Resource getMaximumAllocation() {
return Resources.createResource(mem, cpu);
}
+ public Resource getIncrementAllocation() {
+ Resource minimum = getMinimumAllocation();
+ int incrementMemory = getInt(
+ YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ minimum.getMemory());
+ int incrementCores = getInt(
+ YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES,
+ minimum.getVirtualCores());
+ return Resources.createResource(incrementMemory, incrementCores);
+ }
+
public boolean getUserAsDefaultQueue() {
return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 2f4e70d..f731643 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -109,7 +109,8 @@
private boolean initialized;
private Resource minimumAllocation;
private Resource maximumAllocation;
-
+ private Resource incrementAllocation;
+
private Map applications
= new TreeMap();
@@ -196,6 +197,11 @@ public Resource getMaximumResourceCapability() {
}
@Override
+ public Resource getIncrementResourceCapability() {
+ return incrementAllocation;
+ }
+
+ @Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException
{
@@ -210,6 +216,10 @@ public Resource getMaximumResourceCapability() {
Resources.createResource(conf.getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+ this.incrementAllocation =
+ Resources.createResource(conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ minimumAllocation.getMemory()));
this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
conf);
this.activeUsersManager = new ActiveUsersManager(metrics);
@@ -232,7 +242,8 @@ public Allocation allocate(
// Sanity check
SchedulerUtils.normalizeRequests(ask, resourceCalculator,
- clusterResource, minimumAllocation, maximumAllocation);
+ clusterResource, minimumAllocation, maximumAllocation,
+ incrementAllocation);
// Release containers
for (ContainerId releasedContainer : release) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index f17da43..fcb4866 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -42,45 +42,47 @@ public void testNormalizeRequest() {
final int minMemory = 1024;
final int maxMemory = 8192;
+ final int incrMemory = 256;
Resource minResource = Resources.createResource(minMemory, 0);
Resource maxResource = Resources.createResource(maxMemory, 0);
+ Resource incrResource = Resources.createResource(incrMemory, 0);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory
ask.setCapability(Resources.createResource(-1024));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(minMemory, ask.getCapability().getMemory());
// case zero memory
ask.setCapability(Resources.createResource(0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(minMemory, ask.getCapability().getMemory());
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
- assertEquals(2 * minMemory, ask.getCapability().getMemory());
+ maxResource, incrResource);
+ assertEquals(minMemory + incrMemory, ask.getCapability().getMemory());
// case memory is equal to max allowed
ask.setCapability(Resources.createResource(maxMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(maxMemory, ask.getCapability().getMemory());
// case memory is just less than max
ask.setCapability(Resources.createResource(maxMemory - 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(maxMemory, ask.getCapability().getMemory());
// max is not a multiple of min
@@ -88,14 +90,14 @@ public void testNormalizeRequest() {
ask.setCapability(Resources.createResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
- maxResource);
+ maxResource, incrResource);
assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}
@@ -105,6 +107,7 @@ public void testNormalizeRequestWithDominantResourceCalculator() {
Resource minResource = Resources.createResource(1024, 1);
Resource maxResource = Resources.createResource(10240, 10);
+ Resource incrResource = Resources.createResource(256, 2);
Resource clusterResource = Resources.createResource(10 * 1024, 10);
ResourceRequest ask = new ResourceRequestPBImpl();
@@ -112,24 +115,27 @@ public void testNormalizeRequestWithDominantResourceCalculator() {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ incrResource);
assertEquals(minResource, ask.getCapability());
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ incrResource);
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemory());
// case non-zero memory & zero cores
- ask.setCapability(Resources.createResource(1536, 0));
+ ask.setCapability(Resources.createResource(1200, 0));
SchedulerUtils.normalizeRequest(
- ask, resourceCalculator, clusterResource, minResource, maxResource);
- assertEquals(Resources.createResource(2048, 1), ask.getCapability());
+ ask, resourceCalculator, clusterResource, minResource, maxResource,
+ incrResource);
+ assertEquals(Resources.createResource(1280, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
- assertEquals(2048, ask.getCapability().getMemory());
+ assertEquals(1280, ask.getCapability().getMemory());
}
@Test (timeout = 30000)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 6e37df4..10173c0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -374,6 +374,28 @@ public void testParseQueue() throws IOException {
}
@Test
+ public void testMinMaxIncrResourcesSettings() throws IOException {
+ CapacityScheduler cs = new CapacityScheduler();
+ YarnConfiguration conf = new YarnConfiguration();
+ cs.setConf(conf);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 10);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 3);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory());
+ Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, cs.getIncrementResourceCapability().getMemory());
+ Assert.assertEquals(10, cs.getMaximumResourceCapability().getVirtualCores());
+ Assert.assertEquals(2, cs.getMinimumResourceCapability().getVirtualCores());
+ Assert.assertEquals(3, cs.getIncrementResourceCapability().getVirtualCores());
+ }
+
+ @Test
public void testReconnectedNode() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index e2c753f..b055be6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -236,6 +236,7 @@ public void testLoadConfigurationOnInitialize() throws IOException {
conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128);
scheduler.reinitialize(conf, resourceManager.getRMContext());
Assert.assertEquals(true, scheduler.assignMultiple);
Assert.assertEquals(3, scheduler.maxAssign);
@@ -244,6 +245,7 @@ public void testLoadConfigurationOnInitialize() throws IOException {
Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01);
Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory());
Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, scheduler.getIncrementResourceCapability().getMemory());
}
@Test
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index d252421..3b6224b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -56,6 +56,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
@@ -114,6 +115,22 @@ private ResourceRequest createResourceRequest(int memory, String host,
return request;
}
+ @Test
+ public void testMinMaxIncrResourcesSettings() throws IOException {
+ FifoScheduler cs = new FifoScheduler();
+ YarnConfiguration conf = new YarnConfiguration();
+ cs.setConf(conf);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ Assert.assertEquals(1024, cs.getMaximumResourceCapability().getMemory());
+ Assert.assertEquals(512, cs.getMinimumResourceCapability().getMemory());
+ Assert.assertEquals(128, cs.getIncrementResourceCapability().getMemory());
+ }
+
@Test(timeout=5000)
public void testFifoSchedulerCapacityWhenNoNMs() {
FifoScheduler scheduler = new FifoScheduler();