diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 65b5dce019e..cef1b77ce20 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -72,6 +72,17 @@
@Private
public static final int VCORES_INDEX = 1;
+ /**
+ * Return a new {@link Resource} instance with all resource values
+ * initialized to -1.
+ * @return a new {@link Resource} instance
+ */
+ @Public
+ @Stable
+ public static Resource newInstance() {
+ return new LightWeightResource();
+ }
+
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
index a6e6432976f..d533a782617 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
@@ -65,6 +65,22 @@
private ResourceInformation memoryResInfo;
private ResourceInformation vcoresResInfo;
+ /**
+ * Create a new {@code LightWeightResource} instance with all resource values
+ * initialized to -1.
+ */
+ public LightWeightResource() {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
+
+ initResourceInformations(-1, -1, types.length);
+
+ for (int i = 2; i < types.length; i++) {
+ resources[i] = new ResourceInformation();
+ ResourceInformation.copy(types[i], resources[i]);
+ resources[i].setValue(-1);
+ }
+ }
+
public LightWeightResource(long memory, int vcores) {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 597af9411c0..f27560b1b76 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -58,6 +58,7 @@
import org.xml.sax.SAXException;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
@Public
@Unstable
@@ -499,7 +500,7 @@ private void loadQueue(String parentName, Element element,
String text = ((Text)field.getFirstChild()).getData().trim();
ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text);
- minQueueResources.put(queueName, val.getResource());
+ minQueueResources.put(queueName, val.getResource(0L));
} else if ("maxResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
ConfigurableResource val =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
index ecdd0111a6b..6295d6a929c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
/**
* A {@code ConfigurableResource} object represents an entity that is used to
@@ -46,21 +47,43 @@ public ConfigurableResource(Resource resource) {
/**
* Get resource by multiplying the cluster resource and the percentage of
* each resource respectively. Return the absolute resource if either
- * {@code percentages} or {@code clusterResource} is null.
+ * {@code percentages} or {@code clusterResource} is null with
+ * {@link Long#MAX_VALUE} used to replace any resource types with the
+ * value -1.
*
* @param clusterResource the cluster resource
- * @return resource
+ * @return resource the resulting resource
*/
public Resource getResource(Resource clusterResource) {
if (percentages != null && clusterResource != null) {
long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
- return Resource.newInstance(memory, vcore);
+ Resource res = Resource.newInstance(memory, vcore);
+ ResourceInformation[] clusterInfo = clusterResource.getResources();
+
+ for (int i = 2; i < clusterInfo.length; i++) {
+ res.setResourceValue(i,
+ (long)(clusterInfo[i].getValue() * percentages[i]));
+ }
+
+ return res;
} else {
- return resource;
+ return fillInMissingValues(resource, Long.MAX_VALUE);
}
}
+ private Resource fillInMissingValues(Resource res, long sub) {
+ ResourceInformation[] infos = res.getResources();
+
+ for (ResourceInformation info : infos) {
+ if (info.getValue() == -1) {
+ info.setValue(sub);
+ }
+ }
+
+ return res;
+ }
+
/**
* Get the absolute resource.
*
@@ -69,4 +92,15 @@ public Resource getResource(Resource clusterResource) {
public Resource getResource() {
return resource;
}
+
+ /**
+ * Get the absolute resource with the {@code sub}
+ * value used to replace any resource types with the value -1.
+ *
+ * @param sub the value with which to replace types with the value -1
+ * @return the absolute resource
+ */
+ public Resource getResource(long sub) {
+ return fillInMissingValues(resource, sub);
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 9c9eee627a5..21b7fdc51d2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -283,22 +284,76 @@ public float getReservableNodes() {
}
/**
- * Parses a resource config value of a form like "1024", "1024 mb",
- * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed.
- *
- * @throws AllocationConfigurationException
+ * Parses a resource config value in one of three forms:
+ *
+ * - Percentage: "50%" or "40% memory, 60% cpu"
+ * - New style resources: "vcores=10, memory-mb=1024"
+ * - Old style resources: "1024 mb, 10 vcores"
+ *
+ * In new style resources, any resource that is not specified will be
+ * set to -1, so care must be taken to replace that value with a
+ * context-appropriate value. Also, in the new style resources, units are not
+ * allowed. Units are assumed from the resource manager's settings for the
+ * resources.
+ *
+ * @param value the resource definition to parse
+ * @return a {@link ConfigurableResource} that represents the parsed value
+ * @throws AllocationConfigurationException if the raw value is not a valid
+ * resource definition
*/
- public static ConfigurableResource parseResourceConfigValue(String val)
+ public static ConfigurableResource parseResourceConfigValue(String value)
throws AllocationConfigurationException {
ConfigurableResource configurableResource;
+
+ if (value.trim().isEmpty()) {
+ throw new AllocationConfigurationException("Error reading resource "
+ + "config--the resource string is empty.");
+ }
+
try {
- val = StringUtils.toLowerCase(val);
- if (val.contains("%")) {
+ if (value.contains("%")) {
+ // Resource as a percentage
configurableResource = new ConfigurableResource(
- getResourcePercentage(val));
+ getResourcePercentage(StringUtils.toLowerCase(value)));
+ } else if (value.contains("=")) {
+ // New style resource definiton
+ Resource result = Resource.newInstance();
+ String[] resources = value.split(",");
+
+ for (String resource : resources) {
+ String[] parts = resource.split("=");
+
+ if (parts.length != 2) {
+ throw new AllocationConfigurationException("Error reading resource "
+ + "config--invalid resource definition: " + value + ". Every "
+ + "resource must be of the form: name=value.");
+ }
+
+ try {
+ result.setResourceValue(parts[0].trim(),
+ Long.parseLong(parts[1].trim()));
+ } catch (ResourceNotFoundException ex) {
+ throw new AllocationConfigurationException("Error reading resource "
+ + "config--invalid resource definition: " + value + ". The "
+ + "resource name, \"" + parts[0].trim() + "\" was not "
+ + "recognized. Please check the value of "
+ + YarnConfiguration.RESOURCE_TYPES + " in the resource "
+ + "manager's configuration files.");
+ } catch (NumberFormatException ex) {
+ throw new AllocationConfigurationException("Error reading resource "
+ + "config--invalid resource definition: " + value + ". The "
+ + "resource values must all be integers. \"" + parts[1].trim()
+ + "\" is not an integer.");
+ }
+ }
+
+ configurableResource = new ConfigurableResource(result);
} else {
- int memory = findResource(val, "mb");
- int vcores = findResource(val, "vcores");
+ // Old style resource definition
+ String lCaseValue = StringUtils.toLowerCase(value);
+ int memory = findResource(lCaseValue, "mb");
+ int vcores = findResource(lCaseValue, "vcores");
+
configurableResource = new ConfigurableResource(
BuilderUtils.newResource(memory, vcores));
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 999aaae2ca6..59f89499060 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -17,70 +17,113 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
import static org.junit.Assert.assertEquals;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Test;
public class TestFairSchedulerConfiguration {
@Test
public void testParseResourceConfigValue() throws Exception {
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2 vcores, 1024 mb").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2vcores,1024mb").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024mb,2vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource());
+ Resource expected = BuilderUtils.newResource(5 * 1024, 2);
+ Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4);
- Resource clusterResource = BuilderUtils.newResource(2048, 4);
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
+ parseResourceConfigValue("2 vcores, 5120 mb").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120 mb, 2 vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("2vcores,5120mb").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120mb,2vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120mb mb, 2 vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120 Mb, 2 vCores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource());
+
+ assertEquals(expected,
parseResourceConfigValue("50% memory, 50% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50% Memory, 50% CpU").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("50%").getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 4),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 4),
parseResourceConfigValue("50% memory, 100% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 4),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 4),
parseResourceConfigValue(" 100% cpu, 50% memory").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 0),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 0),
parseResourceConfigValue("50% memory, 0% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50 % memory, 50 % cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50%memory,50%cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue(" 50 % memory, 50 % cpu ").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50.% memory, 50.% cpu").
getResource(clusterResource));
-
- clusterResource = BuilderUtils.newResource(1024 * 10, 4);
assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
parseResourceConfigValue("10.9% memory, 50.6% cpu").
getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("50%").getResource(clusterResource));
+
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RESOURCE_TYPES, "test1");
+ ResourceUtils.resetResourceTypes(conf);
+
+ clusterResource = BuilderUtils.newResource(10 * 1024, 4);
+ expected = BuilderUtils.newResource(5 * 1024, 2);
+ expected.setResourceValue("test1", -1);
+
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2, memory-mb=5120").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=5120, vcores=2").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2,memory-mb=5120").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ")
+ .getResource());
+
+ clusterResource.setResourceValue("test1", 8);
+ expected.setResourceValue("test1", 4);
+
+ assertEquals(expected,
+ parseResourceConfigValue("50%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2, memory-mb=5120, test1=4")
+ .getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("test1=4, vcores=2, memory-mb=5120")
+ .getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=5120, test1=4, vcores=2")
+ .getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2,memory-mb=5120,test1=4")
+ .getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , test1 = 4 ")
+ .getResource());
}
@Test(expected = AllocationConfigurationException.class)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 8d53e57e733..0a7747fbc49 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -83,9 +83,9 @@ The allocation file must be in XML format. The format contains five types of ele
* **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties:
- * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
+ * **minResources**: minimum resources the queue is entitled to, in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
- * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit.
+ * **maxResources**: maximum resources a queue is allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. A queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.