diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 3116ad669f4..8481f63ecc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -216,6 +216,10 @@ private static final String INVALID_RESOURCE_DEFINITION_PREFIX = "Error reading resource config--invalid resource definition: "; + private static final String RESOURCE_PERCENTAGE_PATTERN = + "^(-?(\\d+)(\\.\\d*)?)\\s*%\\s*"; + private static final String RESOURCE_VALUE_PATTERN = + "^(-?\\d+)(\\.\\d*)?\\s*"; public FairSchedulerConfiguration() { super(); @@ -507,7 +511,7 @@ private static ConfigurableResource parseNewStyleResource(String value, try { if (asPercent) { double percentage = parseNewStyleResourceAsPercentage(value, - resourceValue); + resourceName, resourceValue); configurableResource.setPercentage(resourceName, percentage); } else { long parsedValue = parseNewStyleResourceAsAbsoluteValue(value, @@ -526,10 +530,10 @@ private static ConfigurableResource parseNewStyleResource(String value, } private static double parseNewStyleResourceAsPercentage( - String value, String resourceValue) + String value, String resource, String resourceValue) throws AllocationConfigurationException { try { - return findPercentage(resourceValue, ""); + return findPercentage(resourceValue, resource); } catch (AllocationConfigurationException ex) { throw createConfigException(value, "The resource values must all be percentages. \"" @@ -565,16 +569,16 @@ private static ConfigurableResource parseOldStyleResourceAsPercentage( private static ConfigurableResource parseOldStyleResource(String value) throws AllocationConfigurationException { - final String lCaseValue = StringUtils.toLowerCase(value); - final int memory = parseOldStyleResourceMemory(lCaseValue); - final int vcores = parseOldStyleResourceVcores(lCaseValue); + final String[] resources = StringUtils.toLowerCase(value).split(","); + final int memory = parseOldStyleResourceMemory(resources); + final int vcores = parseOldStyleResourceVcores(resources); return new ConfigurableResource( BuilderUtils.newResource(memory, vcores)); } - private static int parseOldStyleResourceMemory(String lCaseValue) + private static int parseOldStyleResourceMemory(String[] resources) throws AllocationConfigurationException { - final int memory = findResource(lCaseValue, "mb"); + final int memory = findResource(resources, "mb"); if (memory < 0) { throw new AllocationConfigurationException( @@ -584,9 +588,9 @@ private static int parseOldStyleResourceMemory(String lCaseValue) return memory; } - private static int parseOldStyleResourceVcores(String lCaseValue) + private static int parseOldStyleResourceVcores(String[] resources) throws AllocationConfigurationException { - final int vcores = findResource(lCaseValue, "vcores"); + final int vcores = findResource(resources, "vcores"); if (vcores < 0) { throw new AllocationConfigurationException( @@ -596,45 +600,62 @@ private static int parseOldStyleResourceVcores(String lCaseValue) return vcores; } - private static double[] getResourcePercentage( - String val) throws AllocationConfigurationException { + private static double[] getResourcePercentage(String val) + throws AllocationConfigurationException { int numberOfKnownResourceTypes = ResourceUtils .getNumberOfKnownResourceTypes(); double[] resourcePercentage = new double[numberOfKnownResourceTypes]; - String[] strings = val.split(","); + String[] values = val.split(","); - if (strings.length == 1) { - double percentage = findPercentage(strings[0], ""); + if (values.length == 1) { + double percentage = findPercentage(values, ""); for (int i = 0; i < numberOfKnownResourceTypes; i++) { resourcePercentage[i] = percentage; } } else { - resourcePercentage[0] = findPercentage(val, "memory"); - resourcePercentage[1] = findPercentage(val, "cpu"); + resourcePercentage[0] = findPercentage(values, "memory"); + resourcePercentage[1] = findPercentage(values, "cpu"); } return resourcePercentage; } - private static double findPercentage(String val, String units) + private static double findPercentage(String resourceValue, String resource) throws AllocationConfigurationException { - final Pattern pattern = - Pattern.compile("(-?(\\d+)(\\.\\d*)?)\\s*%\\s*" + units); - Matcher matcher = pattern.matcher(val); - if (!matcher.find()) { - if (units.equals("")) { + return findPercentageInternal(resource, resourceValue, false); + } + + private static double findPercentage(String[] resourceValues, String resource) + throws AllocationConfigurationException { + String resourceValue = findResourceFromValues(resourceValues, resource); + return findPercentageInternal(resource, resourceValue, true); + } + + private static double findPercentageInternal(String resource, + String resourceValue, boolean includeResourceInPattern) + throws AllocationConfigurationException { + final Pattern pattern; + if (includeResourceInPattern) { + pattern = Pattern.compile(RESOURCE_PERCENTAGE_PATTERN + resource); + } else { + pattern = Pattern.compile(RESOURCE_PERCENTAGE_PATTERN); + } + + Matcher matcher = pattern.matcher(resourceValue); + if (!matcher.matches()) { + if (resource.equals("")) { throw new AllocationConfigurationException("Invalid percentage: " + - val); + resourceValue); } else { - throw new AllocationConfigurationException("Missing resource: " + - units); + throw new AllocationConfigurationException("Invalid percentage of " + + resource + ": " + resourceValue); } } double percentage = Double.parseDouble(matcher.group(1)) / 100.0; if (percentage < 0) { throw new AllocationConfigurationException("Invalid percentage: " + - val + ", percentage should not be negative!"); + resourceValue + ", percentage should not be negative!"); } return percentage; @@ -659,13 +680,26 @@ public long getUpdateInterval() { return getLong(UPDATE_INTERVAL_MS, DEFAULT_UPDATE_INTERVAL_MS); } - private static int findResource(String val, String units) + private static int findResource(String[] resourceValues, String resource) throws AllocationConfigurationException { - final Pattern pattern = Pattern.compile("(-?\\d+)(\\.\\d*)?\\s*" + units); - Matcher matcher = pattern.matcher(val); + String resourceValue = findResourceFromValues(resourceValues, resource); + final Pattern pattern = Pattern.compile(RESOURCE_VALUE_PATTERN + + resource); + Matcher matcher = pattern.matcher(resourceValue); if (!matcher.find()) { - throw new AllocationConfigurationException("Missing resource: " + units); + throw new AllocationConfigurationException("Invalid value of " + + resource + ": " + resourceValue); } return Integer.parseInt(matcher.group(1)); } + + private static String findResourceFromValues(String[] resourceValues, + String resource) throws AllocationConfigurationException { + for (String resourceValue : resourceValues) { + if (resourceValue.contains(resource)) { + return resourceValue.trim(); + } + } + throw new AllocationConfigurationException("Missing resource: " + resource); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index cf7b2e1ae49..f4fafe4ccbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -113,6 +113,16 @@ private void expectMissingResource(String resource) { exception.expectMessage("Missing resource: " + resource); } + private void expectInvalidResource(String resource) { + exception.expect(AllocationConfigurationException.class); + exception.expectMessage("Invalid value of " + resource + ": "); + } + + private void expectInvalidResourcePercentage(String resource) { + exception.expect(AllocationConfigurationException.class); + exception.expectMessage("Invalid percentage of " + resource + ": "); + } + private void expectNegativePercentageOldStyle() { exception.expect(AllocationConfigurationException.class); exception.expectMessage("percentage should not be negative"); @@ -303,7 +313,7 @@ public void testNoUnitsPercentage() throws Exception { @Test public void testInvalidNumPercentage() throws Exception { - expectMissingResource("cpu"); + expectInvalidResourcePercentage("cpu"); parseResourceConfigValue("95A% cpu, 50% memory"); } @@ -319,6 +329,44 @@ public void testMemoryPercentageCpuAbsolute() throws Exception { parseResourceConfigValue("50% memory, 2 vcores"); } + @Test + public void testDuplicateVcoresDefinitionAbsolute() throws Exception { + expectInvalidResource("vcores"); + parseResourceConfigValue("1024 mb, 2 4 vcores"); + } + + @Test + public void testDuplicateMemoryDefinitionAbsolute() throws Exception { + expectInvalidResource("mb"); + parseResourceConfigValue("2048 1024 mb, 2 vcores"); + } + + @Test + public void testDuplicateVcoresDefinitionPercentage() throws Exception { + expectInvalidResourcePercentage("cpu"); + parseResourceConfigValue("50% memory, 50% 100%cpu"); + } + + @Test + public void testDuplicateMemoryDefinitionPercentage() throws Exception { + expectInvalidResourcePercentage("memory"); + parseResourceConfigValue("50% 80% memory, 100%cpu"); + } + + @Test + public void testParseNewStyleDuplicateMemoryDefinitionPercentage() + throws Exception { + expectInvalidResourcePercentage("memory-mb"); + parseResourceConfigValue("vcores = 75%, memory-mb = 40% 80%"); + } + + @Test + public void testParseNewStyleDuplicateVcoresDefinitionPercentage() + throws Exception { + expectInvalidResourcePercentage("vcores"); + parseResourceConfigValue("vcores = 75% 65%, memory-mb = 40%"); + } + @Test public void testMemoryPercentageNegativeValue() throws Exception { expectNegativePercentageOldStyle();