From be5234e601037666912b585b4cff8ee631d61de1 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Tue, 26 Nov 2019 14:08:46 +0100 Subject: [PATCH] YARN-9444 YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource --- .../yarn/util/resource/ResourceUtils.java | 3 +- .../yarn/util/resource/TestResourceUtils.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 9aaff15f469..a86db15116c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -76,6 +76,7 @@ + "\\p{Alnum}([\\p{Alnum}-]*\\p{Alnum})?)/)?\\p{Alpha}([\\w.-]*)$"); private final static String RES_PATTERN = "^[^=]+=\\d+\\s?\\w*$"; + public static final String YARN_IO_OPTIONAL = "(yarn\\.io/)?"; private static volatile boolean initializedResources = false; private static final Map RESOURCE_NAME_TO_INDEX = @@ -696,7 +697,7 @@ public static void reinitializeResources( Configuration configuration, String prefix) { List result = new ArrayList<>(); Map customResourcesMap = configuration - .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$"); + .getValByRegex("^" + Pattern.quote(prefix) + YARN_IO_OPTIONAL + "[^.]+$"); for (Entry resource : customResourcesMap.entrySet()) { String resourceName = resource.getKey().substring(prefix.length()); Matcher matcher = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 9cc96b62e4d..42df5876bd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -41,7 +41,9 @@ import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -191,6 +193,47 @@ public void testGetResourceTypesConfigs() throws Exception { } } + @Test + public void testGetRequestedResourcesFromConfig() { + Configuration conf = new Configuration(); + + //these resource type configurations should be recognised + String propertyPrefix = "mapreduce.mapper.proper.rt."; + String []expectedKeys = { + "yarn.io/gpu", + "yarn.io/fpga", + "yarn.io/anything_without_a_dot", + "regular_rt", + "regular_rt/with_slash"}; + + String []invalidKeys = { + propertyPrefix + "too.many_parts", + propertyPrefix + "yarn.notio/gpu", + "incorrect.prefix.yarn.io/gpu", + propertyPrefix + "yarn.io/", + propertyPrefix}; + + for (String s : expectedKeys) { + //setting the properties which are expected to be in the resource list + conf.set(propertyPrefix + s, "42"); + } + + for (String s : invalidKeys) { + //setting the properties which are expected to be in the resource list + conf.set(s, "24"); + } + + List properList = + ResourceUtils.getRequestedResourcesFromConfig(conf, propertyPrefix); + HashSet expectedSet = + new HashSet(Arrays.asList(expectedKeys)); + + Assert.assertEquals(properList.size(), expectedKeys.length); + properList.stream().forEach( + item -> Assert.assertTrue(expectedSet.contains(item.getName()))); + + } + @Test public void testGetResourceTypesConfigErrors() throws IOException { Configuration conf = new YarnConfiguration(); -- 2.20.1 (Apple Git-117)