diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index ecdd0111a6b..1c1bfe226c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** * A {@code ConfigurableResource} object represents an entity that is used to @@ -49,13 +51,21 @@ public ConfigurableResource(Resource resource) { * {@code percentages} or {@code clusterResource} is null. * * @param clusterResource the cluster resource - * @return resource + * @return resource the resulting resource */ public Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); - return Resource.newInstance(memory, vcore); + Resource res = Resource.newInstance(memory, vcore); + ResourceInformation[] clusterInfo = clusterResource.getResources(); + + for (int i = 2; i < clusterInfo.length; i++) { + res.setResourceValue(i, + (long)(clusterInfo[i].getValue() * percentages[i])); + } + + return res; } else { return resource; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 9c9eee627a5..0d8a6978060 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -29,6 +29,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -283,22 +284,76 @@ public float getReservableNodes() { } /** - * Parses a resource config value of a form like "1024", "1024 mb", - * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed. - * - * @throws AllocationConfigurationException + * Parses a resource config value in one of three forms: + *
    + *
  1. Percentage: "50%" or "40% memory, 60% cpu"
  2. + *
  3. New style resources: "vcores=10, memory-mb=1024"
  4. + *
  5. Old style resources: "1024 mb, 10 vcores"
  6. + *
+ * In new style resources, any resource that is not specified will be + * set to -1, so care must be taken to replace that value with a + * context-appropriate value. Also, in the new style resources, units are not + * allowed. Units are assumed from the resource manager's settings for the + * resources. + * + * @param value the resource definition to parse + * @return a {@link ConfigurableResource} that represents the parsed value + * @throws AllocationConfigurationException if the raw value is not a valid + * resource definition */ - public static ConfigurableResource parseResourceConfigValue(String val) + public static ConfigurableResource parseResourceConfigValue(String value) throws AllocationConfigurationException { ConfigurableResource configurableResource; + + if (value.trim().isEmpty()) { + throw new AllocationConfigurationException("Error reading resource " + + "config--the resource string is empty."); + } + try { - val = StringUtils.toLowerCase(val); - if (val.contains("%")) { + if (value.contains("%")) { + // Resource as a percentage configurableResource = new ConfigurableResource( - getResourcePercentage(val)); + getResourcePercentage(StringUtils.toLowerCase(value))); + } else if (value.contains("=")) { + // New style resource definiton + Resource result = Resource.newInstance(0L, 0); + String[] resources = value.split(","); + + for (String resource : resources) { + String[] parts = resource.split("="); + + if (parts.length != 2) { + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". Every " + + "resource must be of the form: name=value."); + } + + try { + result.setResourceValue(parts[0].trim(), + Long.parseLong(parts[1].trim())); + } catch (ResourceNotFoundException ex) { + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". The " + + "resource name, \"" + parts[0].trim() + "\" was not " + + "recognized. Please check the value of " + + YarnConfiguration.RESOURCE_TYPES + " in the resource " + + "manager's configuration files."); + } catch (NumberFormatException ex) { + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". The " + + "resource values must all be integers. \"" + parts[1].trim() + + "\" is not an integer."); + } + } + + configurableResource = new ConfigurableResource(result); } else { - int memory = findResource(val, "mb"); - int vcores = findResource(val, "vcores"); + // Old style resource definition + String lCaseValue = StringUtils.toLowerCase(value); + int memory = findResource(lCaseValue, "mb"); + int vcores = findResource(lCaseValue, "vcores"); + configurableResource = new ConfigurableResource( BuilderUtils.newResource(memory, vcores)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 999aaae2ca6..b8dbeea1b13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -17,70 +17,115 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import org.apache.hadoop.conf.Configuration; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Test; public class TestFairSchedulerConfiguration { @Test public void testParseResourceConfigValue() throws Exception { - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2 vcores, 1024 mb").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2vcores,1024mb").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024mb,2vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 Mb, 2 vCores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource()); + Resource expected = BuilderUtils.newResource(5 * 1024, 2); + Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4); - Resource clusterResource = BuilderUtils.newResource(2048, 4); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, + parseResourceConfigValue("2 vcores, 5120 mb").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120 mb, 2 vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("2vcores,5120mb").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120mb,2vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120mb mb, 2 vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120 Mb, 2 vCores").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource()); + + assertEquals(expected, parseResourceConfigValue("50% memory, 50% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50% Memory, 50% CpU"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("50%").getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 4), + assertEquals(BuilderUtils.newResource(5 * 1024, 4), parseResourceConfigValue("50% memory, 100% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 4), + assertEquals(BuilderUtils.newResource(5 * 1024, 4), parseResourceConfigValue(" 100% cpu, 50% memory"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 0), + assertEquals(BuilderUtils.newResource(5 * 1024, 0), parseResourceConfigValue("50% memory, 0% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50 % memory, 50 % cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50%memory,50%cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue(" 50 % memory, 50 % cpu "). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50.% memory, 50.% cpu"). getResource(clusterResource)); - - clusterResource = BuilderUtils.newResource(1024 * 10, 4); assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2), parseResourceConfigValue("10.9% memory, 50.6% cpu"). getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("50%").getResource(clusterResource)); + + Configuration conf = new Configuration(); + + conf.set(YarnConfiguration.RESOURCE_TYPES, "test1"); + ResourceUtils.resetResourceTypes(conf); + + clusterResource = BuilderUtils.newResource(10 * 1024, 4); + expected = BuilderUtils.newResource(5 * 1024, 2); + + assertEquals(expected, + parseResourceConfigValue("vcores=2, memory-mb=5120") + .getResource()); + assertEquals(expected, + parseResourceConfigValue("memory-mb=5120, vcores=2") + .getResource()); + assertEquals(expected, + parseResourceConfigValue("vcores=2,memory-mb=5120") + .getResource()); + assertEquals(expected, + parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ") + .getResource()); + + clusterResource.setResourceValue("test1", 8); + expected.setResourceValue("test1", 4); + + assertEquals(expected, + parseResourceConfigValue("50%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("vcores=2, memory-mb=5120, test1=4") + .getResource()); + assertEquals(expected, + parseResourceConfigValue("test1=4, vcores=2, memory-mb=5120") + .getResource()); + assertEquals(expected, + parseResourceConfigValue("memory-mb=5120, test1=4, vcores=2") + .getResource()); + assertEquals(expected, + parseResourceConfigValue("vcores=2,memory-mb=5120,test1=4") + .getResource()); + assertEquals(expected, + parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , test1 = 4 ") + .getResource()); } @Test(expected = AllocationConfigurationException.class) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index 8d53e57e733..0a7747fbc49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -83,9 +83,9 @@ The allocation file must be in XML format. The format contains five types of ele * **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties: - * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. + * **minResources**: minimum resources the queue is entitled to, in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. - * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit. + * **maxResources**: maximum resources a queue is allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. A queue will not be assigned a container that would put its aggregate usage over this limit. * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.