diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java new file mode 100644 index 0000000..819a607 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * {@code AbstractResourceRequest} represents a generic resource request made + * by an application to the {@code ResourceManager}. + *
+ * It includes: + *
Resource capability of the request
+ * @param capability Resource capability of the request
+ */
+ @Public
+ @Stable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the Resource capability of the request.
+ * @return Resource capability of the request
+ */
+ @Public
+ @Stable
+ public abstract Resource getCapability();
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index be2c783..ce85b21 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -58,7 +58,8 @@
*/
@Public
@Stable
-public abstract class ResourceRequest implements ComparableResource capability of the request.
- * @return Resource capability of the request
- */
- @Public
- @Stable
- public abstract Resource getCapability();
-
- /**
- * Set the Resource capability of the request
- * @param capability Resource capability of the request
- */
- @Public
- @Stable
- public abstract void setCapability(Resource capability);
-
- /**
* Get the number of containers required with the given specifications.
* @return number of containers required with the given specifications
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index ef39f5c..200dea3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -62,7 +62,7 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public abstract class UpdateContainerRequest {
+public abstract class UpdateContainerRequest extends AbstractResourceRequest {
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -128,22 +128,6 @@ public static UpdateContainerRequest newInstance(int version,
public abstract void setContainerId(ContainerId containerId);
/**
- * Get the Resource capability of the container.
- * @return Resource capability of the container
- */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public abstract Resource getCapability();
-
- /**
- * Set the Resource capability of the container.
- * @param capability Resource capability of the container
- */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public abstract void setCapability(Resource capability);
-
- /**
* Get the target ExecutionType of the container.
* @return ExecutionType of the container
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index fe78de2..42c45ad 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -24,7 +26,9 @@
@Private
@Unstable
public class DefaultResourceCalculator extends ResourceCalculator {
-
+ private static final Log LOG =
+ LogFactory.getLog(DefaultResourceCalculator.class);
+
@Override
public int compare(Resource unused, Resource lhs, Resource rhs) {
// Only consider memory
@@ -64,6 +68,13 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
+ if (stepFactor.getMemorySize() == 0) {
+ LOG.error("Memory cannot be allocated in increments of zero. Assuming " +
+ minimumResource.getMemorySize() + "MB increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ stepFactor = minimumResource;
+ }
+
long normalizedMemory = Math.min(
roundUp(
Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
@@ -73,12 +84,6 @@ public Resource normalize(Resource r, Resource minimumResource,
}
@Override
- public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- return normalize(r, minimumResource, maximumResource, minimumResource);
- }
-
- @Override
public Resource roundUp(Resource r, Resource stepFactor) {
return Resources.createResource(
roundUp(r.getMemorySize(), stepFactor.getMemorySize())
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 2695811..9f1c8d7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -45,7 +47,9 @@
@Private
@Unstable
public class DominantResourceCalculator extends ResourceCalculator {
-
+ private static final Log LOG =
+ LogFactory.getLog(DominantResourceCalculator.class);
+
@Override
public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
@@ -152,6 +156,25 @@ public Resource divideAndCeil(Resource numerator, int denominator) {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
+ if (stepFactor.getMemorySize() == 0 || stepFactor.getVirtualCores() == 0) {
+ Resource step = Resources.clone(stepFactor);
+ if (stepFactor.getMemorySize() == 0) {
+ LOG.error("Memory cannot be allocated in increments of zero. Assuming "
+ + minimumResource.getMemorySize() + "MB increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ step.setMemorySize(minimumResource.getMemorySize());
+ }
+
+ if (stepFactor.getVirtualCores() == 0) {
+ LOG.error("VCore cannot be allocated in increments of zero. Assuming "
+ + minimumResource.getVirtualCores() + "VCores increment size. "
+ + "Please ensure the scheduler configuration is correct.");
+ step.setVirtualCores(minimumResource.getVirtualCores());
+ }
+
+ stepFactor = step;
+ }
+
long normalizedMemory = Math.min(
roundUp(
Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index a7df9c5..50ce04c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -99,22 +99,7 @@ public abstract Resource multiplyAndNormalizeDown(
/**
* Normalize resource r given the base
* minimumResource and verify against max allowed
- * maximumResource
- *
- * @param r resource
- * @param minimumResource step-factor
- * @param maximumResource the upper bound of the resource to be allocated
- * @return normalized resource
- */
- public Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource) {
- return normalize(r, minimumResource, maximumResource, minimumResource);
- }
-
- /**
- * Normalize resource r given the base
- * minimumResource and verify against max allowed
- * maximumResource using a step factor for hte normalization.
+ * maximumResource using a step factor for the normalization.
*
* @param r resource
* @param minimumResource minimum value
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index 0654891..b123b05 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -150,4 +150,83 @@ private void assertResourcesOperations(Resource clusterResource,
Resources.min(resourceCalculator, clusterResource, lhs, rhs));
}
+ /**
+ * Test resource normalization.
+ */
+ @Test(timeout = 10000)
+ public void testNormalize() {
+ // requested resources value cannot be an arbitrary number.
+ Resource ask = Resource.newInstance(1111, 2);
+ Resource min = Resource.newInstance(1024, 1);
+ Resource max = Resource.newInstance(8 * 1024, 8);
+ Resource increment = Resource.newInstance(1024, 4);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(4, result.getVirtualCores());
+ }
+
+ // if resources asked are less than minimum resource, then normalize it to
+ // minimum resource.
+ ask = Resource.newInstance(512, 0);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(1024, 1);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(2, result.getVirtualCores());
+ }
+
+ // if resources asked are larger than maximum resource, then normalize it to
+ // maximum resources.
+ ask = Resource.newInstance(9 * 1024, 9);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(1024, 1);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(8 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(8 * 1024, result.getMemorySize());
+ Assert.assertEquals(8, result.getVirtualCores());
+ }
+
+ // if increment is 0, use minimum resource as the increment resource.
+ ask = Resource.newInstance(1111, 2);
+ min = Resource.newInstance(2 * 1024, 2);
+ max = Resource.newInstance(8 * 1024, 8);
+ increment = Resource.newInstance(0, 0);
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Resource result = Resources.normalize(resourceCalculator,
+ ask, min, max, increment);
+
+ Assert.assertEquals(2 * 1024, result.getMemorySize());
+ Assert.assertEquals(2, result.getVirtualCores());
+ }
+ }
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index b5f02fc..ce3da06 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -460,11 +460,7 @@ private ResourceRequest validateAndCreateResourceRequest(
throw e;
}
- SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(),
- scheduler.getClusterResource(),
- scheduler.getMinimumResourceCapability(),
- scheduler.getMaximumResourceCapability(),
- scheduler.getMinimumResourceCapability());
+ scheduler.normalizeRequest(amReq);
return amReq;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 98a34d9..4f33d3b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -73,7 +73,6 @@
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -304,13 +303,7 @@ private static boolean validateIncreaseDecreaseRequest(RMContext rmContext,
return false;
}
ResourceScheduler scheduler = rmContext.getScheduler();
- ResourceCalculator rc = scheduler.getResourceCalculator();
- Resource targetResource = Resources.normalize(rc, request.getCapability(),
- scheduler.getMinimumResourceCapability(),
- scheduler.getMaximumResourceCapability(),
- scheduler.getMinimumResourceCapability());
- // Update normalized target resource
- request.setCapability(targetResource);
+ scheduler.normalizeRequest(request);
return true;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 94af4dd..4818ea3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -51,6 +51,7 @@
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -1024,4 +1025,23 @@ protected synchronized void nodeUpdate(RMNode nm) {
}
}
+ @Override
+ public void normalizeRequest(AbstractResourceRequest ask) {
+ SchedulerUtils.normalizeRequest(ask,
+ getResourceCalculator(),
+ getMinimumResourceCapability(),
+ getMaximumResourceCapability(),
+ getMinimumResourceCapability());
+ }
+
+ /**
+ * Normalize a list of resource requests.
+ *
+ * @param asks resource requests
+ */
+ protected void normalizeRequests(List