diff --git hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 45aa868830c..d6309911a90 100644 --- hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -67,11 +67,6 @@ - - - - - @@ -118,6 +113,18 @@ + + + + + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 2b367bf9ed8..f5d70e0dd82 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; @@ -73,6 +74,18 @@ @Private public static final int VCORES_INDEX = 1; + /** + * Return a new {@link Resource} instance with all resource values + * initialized to {@code value}. + * @param value the value to use for all resources + * @return a new {@link Resource} instance + */ + @Private + @Unstable + public static Resource newInstance(long value) { + return new LightWeightResource(value); + } + @Public @Stable public static Resource newInstance(int memory, int vCores) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java index a6e6432976f..83398559be9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java @@ -18,9 +18,8 @@ package org.apache.hadoop.yarn.api.records.impl; -import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -58,13 +57,30 @@ * * @see Resource */ -@InterfaceAudience.Private +@Private @Unstable public class LightWeightResource extends Resource { private ResourceInformation memoryResInfo; private ResourceInformation vcoresResInfo; + /** + * Create a new {@link LightWeightResource} instance with all resource values + * initialized to {@code value}. + * @param value the value to use for all resources + */ + public LightWeightResource(long value) { + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); + + initResourceInformations(value, value, types.length); + + for (int i = 2; i < types.length; i++) { + resources[i] = new ResourceInformation(); + ResourceInformation.copy(types[i], resources[i]); + resources[i].setValue(value); + } + } + public LightWeightResource(long memory, int vcores) { int numberOfKnownResourceTypes = ResourceUtils .getNumberOfKnownResourceTypes(); @@ -91,7 +107,7 @@ public LightWeightResource(long memory, int vcores, } } - private void initResourceInformations(long memory, int vcores, + private void initResourceInformations(long memory, long vcores, int numberOfKnownResourceTypes) { this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(), memory); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 597af9411c0..7676b0acc73 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -498,7 +498,7 @@ private void loadQueue(String parentName, Element element, if ("minResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); ConfigurableResource val = - FairSchedulerConfiguration.parseResourceConfigValue(text); + FairSchedulerConfiguration.parseResourceConfigValue(text, 0L); minQueueResources.put(queueName, val.getResource()); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index ecdd0111a6b..1e4c7c61370 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -18,9 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import java.util.Arrays; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** * A {@code ConfigurableResource} object represents an entity that is used to @@ -29,33 +33,58 @@ */ @Private @Unstable -public class ConfigurableResource { +class ConfigurableResource { private final Resource resource; private final double[] percentages; - public ConfigurableResource(double[] percentages) { + ConfigurableResource() { + this(getOneHundredPercentArray()); + } + + ConfigurableResource(double[] percentages) { this.percentages = percentages.clone(); this.resource = null; } - public ConfigurableResource(Resource resource) { + ConfigurableResource(long value) { + this(Resource.newInstance(value)); + } + + ConfigurableResource(Resource resource) { this.percentages = null; this.resource = resource; } + private static double[] getOneHundredPercentArray() { + double[] nothing = + new double[ResourceUtils.getNumberOfKnownResourceTypes()]; + + Arrays.fill(nothing, 1.0); + + return nothing; + } + /** * Get resource by multiplying the cluster resource and the percentage of * each resource respectively. Return the absolute resource if either * {@code percentages} or {@code clusterResource} is null. * * @param clusterResource the cluster resource - * @return resource + * @return resource the resulting resource */ - public Resource getResource(Resource clusterResource) { + Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); - return Resource.newInstance(memory, vcore); + Resource res = Resource.newInstance(memory, vcore); + ResourceInformation[] clusterInfo = clusterResource.getResources(); + + for (int i = 2; i < clusterInfo.length; i++) { + res.setResourceValue(i, + (long)(clusterInfo[i].getValue() * percentages[i])); + } + + return res; } else { return resource; } @@ -66,7 +95,40 @@ public Resource getResource(Resource clusterResource) { * * @return the absolute resource */ - public Resource getResource() { + Resource getResource() { return resource; } + + /** + * Set the value of the wrapped resource if this object isn't setup to use + * percentages. + * + * @param name the name of the resource + * @param value the value + */ + void setValue(String name, long value) { + if (resource != null) { + resource.setResourceValue(name, value); + } + } + + /** + * Set the percentage of the resource if this object is setup to use + * percentages. + * + * @param name the name of the resource + * @param value the percentage + */ + void setPercentage(String name, double value) { + if (percentages != null) { + Integer index = ResourceUtils.getResourceTypeIndex().get(name); + + if (index != null) { + percentages[index] = value; + } else { + throw new ResourceNotFoundException("The requested resource, \"" + + name + "\", could not be found."); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 960299b70bf..a14a7d0e221 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -29,8 +29,9 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @Private @@ -283,47 +284,150 @@ public float getReservableNodes() { } /** - * Parses a resource config value of a form like "1024", "1024 mb", - * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed. - * - * @throws AllocationConfigurationException + * Parses a resource config value in one of three forms: + *
    + *
  1. Percentage: "50%" or "40% memory, 60% cpu"
  2. + *
  3. New style resources: "vcores=10, memory-mb=1024" + * or "vcores=60%, memory-mb=40%"
  4. + *
  5. Old style resources: "1024 mb, 10 vcores"
  6. + *
+ * In new style resources, any resource that is not specified will be + * set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new + * style resources, units are not allowed. Units are assumed from the resource + * manager's settings for the resources when the value isn't a percentage. + * + * @param value the resource definition to parse + * @return a {@link ConfigurableResource} that represents the parsed value + * @throws AllocationConfigurationException if the raw value is not a valid + * resource definition */ - public static ConfigurableResource parseResourceConfigValue(String val) + static ConfigurableResource parseResourceConfigValue(String value) throws AllocationConfigurationException { + return parseResourceConfigValue(value, Long.MAX_VALUE); + } + + /** + * Parses a resource config value in one of three forms: + *
    + *
  1. Percentage: "50%" or "40% memory, 60% cpu"
  2. + *
  3. New style resources: "vcores=10, memory-mb=1024" + * or "vcores=60%, memory-mb=40%"
  4. + *
  5. Old style resources: "1024 mb, 10 vcores"
  6. + *
+ * In new style resources, any resource that is not specified will be + * set to {@code missing} or 0%, as appropriate. Also, in the new style + * resources, units are not allowed. Units are assumed from the resource + * manager's settings for the resources when the value isn't a percentage. + * + * The {@code missing} parameter is only used in the case of new style + * resources without percentages. With new style resources with percentages, + * any missing resources will be assumed to be 100% because percentages are + * only used with maximum resource limits. + * + * @param value the resource definition to parse + * @param missing the value to use for any unspecified resources + * @return a {@link ConfigurableResource} that represents the parsed value + * @throws AllocationConfigurationException if the raw value is not a valid + * resource definition + */ + static ConfigurableResource parseResourceConfigValue(String value, + long missing) throws AllocationConfigurationException { ConfigurableResource configurableResource; + + if (value.trim().isEmpty()) { + throw new AllocationConfigurationException("Error reading resource " + + "config--the resource string is empty."); + } + try { - val = StringUtils.toLowerCase(val); - if (val.contains("%")) { + if (value.contains("=")) { + // New style resource definiton + boolean asPercent = value.contains("%"); + + if (asPercent) { + configurableResource = new ConfigurableResource(); + } else { + configurableResource = new ConfigurableResource(missing); + } + + String[] resources = value.split(","); + + for (String resource : resources) { + String[] parts = resource.split("="); + + if (parts.length != 2) { + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". Every " + + "resource must be of the form: name=value."); + } + + try { + if (asPercent) { + configurableResource.setPercentage(parts[0].trim(), + findPercentage(parts[1].trim(), "")); + } else { + configurableResource.setValue(parts[0].trim(), + Long.parseLong(parts[1].trim())); + } + } catch (ResourceNotFoundException ex) { + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". The " + + "resource name, \"" + parts[0].trim() + "\" was not " + + "recognized. Please check the value of " + + YarnConfiguration.RESOURCE_TYPES + " in the resource " + + "manager's configuration files.", ex); + } catch (NumberFormatException ex) { + // This only comes from Long.parseLong() + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". The " + + "resource values must all be integers. \"" + parts[1].trim() + + "\" is not an integer.", ex); + } catch (AllocationConfigurationException ex) { + // This only comes from findPercentage() + throw new AllocationConfigurationException("Error reading resource " + + "config--invalid resource definition: " + value + ". The " + + "resource values must all be percentages. \"" + + parts[1].trim() + "\" is either not a number or does not " + + "include the '%' symbol.", ex); + } + } + } else if (value.contains("%")) { + // Old style resource as a percentage configurableResource = new ConfigurableResource( - getResourcePercentage(val)); + getResourcePercentage(StringUtils.toLowerCase(value))); } else { - int memory = findResource(val, "mb"); - int vcores = findResource(val, "vcores"); + // Old style resource definition + String lCaseValue = StringUtils.toLowerCase(value); + int memory = findResource(lCaseValue, "mb"); + int vcores = findResource(lCaseValue, "vcores"); + configurableResource = new ConfigurableResource( BuilderUtils.newResource(memory, vcores)); } - } catch (AllocationConfigurationException ex) { - throw ex; - } catch (Exception ex) { + } catch (RuntimeException ex) { throw new AllocationConfigurationException( "Error reading resource config", ex); } + return configurableResource; } private static double[] getResourcePercentage( String val) throws AllocationConfigurationException { - double[] resourcePercentage = new double[ResourceType.values().length]; + int n = ResourceUtils.getNumberOfKnownResourceTypes(); + double[] resourcePercentage = new double[n]; String[] strings = val.split(","); + if (strings.length == 1) { double percentage = findPercentage(strings[0], ""); - for (int i = 0; i < ResourceType.values().length; i++) { - resourcePercentage[i] = percentage/100; + for (int i = 0; i < n; i++) { + resourcePercentage[i] = percentage; } } else { - resourcePercentage[0] = findPercentage(val, "memory")/100; - resourcePercentage[1] = findPercentage(val, "cpu")/100; + resourcePercentage[0] = findPercentage(val, "memory"); + resourcePercentage[1] = findPercentage(val, "cpu"); } + return resourcePercentage; } @@ -341,7 +445,7 @@ private static double findPercentage(String val, String units) units); } } - return Double.parseDouble(matcher.group(1)); + return Double.parseDouble(matcher.group(1)) / 100.0; } public long getUpdateInterval() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 999aaae2ca6..d4da436764f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -20,67 +20,155 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Test; public class TestFairSchedulerConfiguration { @Test public void testParseResourceConfigValue() throws Exception { - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2 vcores, 1024 mb").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2vcores,1024mb").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024mb,2vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 Mb, 2 vCores").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource()); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource()); - - Resource clusterResource = BuilderUtils.newResource(2048, 4); - assertEquals(BuilderUtils.newResource(1024, 2), + Resource expected = BuilderUtils.newResource(5 * 1024, 2); + Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4); + + assertEquals(expected, + parseResourceConfigValue("2 vcores, 5120 mb").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120 mb, 2 vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("2vcores,5120mb").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120mb,2vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120mb mb, 2 vcores").getResource()); + assertEquals(expected, + parseResourceConfigValue("5120 Mb, 2 vCores").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource()); + assertEquals(expected, + parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource()); + + assertEquals(expected, parseResourceConfigValue("50% memory, 50% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50% Memory, 50% CpU"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("50%").getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 4), + assertEquals(BuilderUtils.newResource(5 * 1024, 4), parseResourceConfigValue("50% memory, 100% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 4), + assertEquals(BuilderUtils.newResource(5 * 1024, 4), parseResourceConfigValue(" 100% cpu, 50% memory"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 0), + assertEquals(BuilderUtils.newResource(5 * 1024, 0), parseResourceConfigValue("50% memory, 0% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50 % memory, 50 % cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50%memory,50%cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue(" 50 % memory, 50 % cpu "). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(1024, 2), + assertEquals(expected, parseResourceConfigValue("50.% memory, 50.% cpu"). getResource(clusterResource)); - - clusterResource = BuilderUtils.newResource(1024 * 10, 4); assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2), parseResourceConfigValue("10.9% memory, 50.6% cpu"). getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("50%").getResource(clusterResource)); + + Configuration conf = new Configuration(); + + conf.set(YarnConfiguration.RESOURCE_TYPES, "test1"); + ResourceUtils.resetResourceTypes(conf); + + clusterResource = BuilderUtils.newResource(10 * 1024, 4); + expected = BuilderUtils.newResource(5 * 1024, 2); + expected.setResourceValue("test1", Long.MAX_VALUE); + + assertEquals(expected, + parseResourceConfigValue("vcores=2, memory-mb=5120").getResource()); + assertEquals(expected, + parseResourceConfigValue("memory-mb=5120, vcores=2").getResource()); + assertEquals(expected, + parseResourceConfigValue("vcores=2,memory-mb=5120").getResource()); + assertEquals(expected, parseResourceConfigValue(" vcores = 2 , " + + "memory-mb = 5120 ").getResource()); + + expected.setResourceValue("test1", 0L); + + assertEquals(expected, + parseResourceConfigValue("vcores=2, memory-mb=5120", 0L).getResource()); + assertEquals(expected, + parseResourceConfigValue("memory-mb=5120, vcores=2", 0L).getResource()); + assertEquals(expected, + parseResourceConfigValue("vcores=2,memory-mb=5120", 0L).getResource()); + assertEquals(expected, + parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ", + 0L).getResource()); + + clusterResource.setResourceValue("test1", 8L); + expected.setResourceValue("test1", 4L); + + assertEquals(expected, + parseResourceConfigValue("50%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("vcores=2, memory-mb=5120, " + + "test1=4").getResource()); + assertEquals(expected, + parseResourceConfigValue("test1=4, vcores=2, " + + "memory-mb=5120").getResource()); + assertEquals(expected, + parseResourceConfigValue("memory-mb=5120, test1=4, " + + "vcores=2").getResource()); + assertEquals(expected, + parseResourceConfigValue("vcores=2,memory-mb=5120," + + "test1=4").getResource()); + assertEquals(expected, + parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , " + + "test1 = 4 ").getResource()); + + expected = BuilderUtils.newResource(4 * 1024, 3); + expected.setResourceValue("test1", 8L); + + assertEquals(expected, + parseResourceConfigValue("vcores=75%, " + + "memory-mb=40%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("memory-mb=40%, " + + "vcores=75%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("vcores=75%," + + "memory-mb=40%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue(" vcores = 75 % , " + + "memory-mb = 40 % ").getResource(clusterResource)); + + expected.setResourceValue("test1", 4L); + + assertEquals(expected, + parseResourceConfigValue("vcores=75%, memory-mb=40%, " + + "test1=50%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("test1=50%, vcores=75%, " + + "memory-mb=40%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("memory-mb=40%, test1=50%, " + + "vcores=75%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue("vcores=75%,memory-mb=40%," + + "test1=50%").getResource(clusterResource)); + assertEquals(expected, + parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , " + + "test1 = 50 % ").getResource(clusterResource)); } @Test(expected = AllocationConfigurationException.class) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index e59f86bb51a..acce6c3b6d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -83,9 +83,9 @@ The allocation file must be in XML format. The format contains five types of ele * **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties: - * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. + * **minResources**: minimum resources the queue is entitled to, in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. - * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit. + * **maxResources**: maximum resources a queue is allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit. * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.