From ddc37842aa896d47be41dca90387010d88241c8e Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Fri, 5 Apr 2019 14:11:36 +0200 Subject: [PATCH] YARN-9444. Fixing resource utils to accept yarn.io/* as valid resource types --- .../yarn/util/resource/ResourceUtils.java | 2 +- .../yarn/util/resource/TestResourceUtils.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 13a872cb414..27734b17958 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -675,7 +675,7 @@ public static void reinitializeResources( Configuration configuration, String prefix) { List result = new ArrayList<>(); Map customResourcesMap = configuration - .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$"); + .getValByRegex("^" + Pattern.quote(prefix) + "(yarn\\.io/)?[^.]+$"); for (Entry resource : customResourcesMap.entrySet()) { String resourceName = resource.getKey().substring(prefix.length()); Matcher matcher = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index f7ec4f803f2..c2735521eaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -139,6 +139,30 @@ public void testGetResourceTypesConfigs() throws Exception { } } + @Test + public void testGetRequestedResourcesFromConfig() { + Configuration conf = new Configuration(); + + //these resource type configurations should be recognised + conf.set("mapreduce.mapper.proper.rt.yarn.io/gpu", "23"); + conf.set("mapreduce.mapper.proper.rt.yarn.io/fpga", "23"); + conf.set("mapreduce.mapper.proper.rt.yarn.io/anything_without_a_dot", "23"); + conf.set("mapreduce.mapper.proper.rt.regular_rt", "23"); + conf.set("mapreduce.mapper.proper.rt.regular_rt/with_slash", "23"); + + //these shouldnt + conf.set("mapreduce.mapper.incorrect.rt.too.many_parts", "23"); + conf.set("mapreduce.mapper.incorrect.rt.yarn.notio/gpu", "23"); + conf.set("incorrect.prefix.yarn.io/gpu", "23"); + conf.set("mapreduce.mapper.incorrect.rt.", "23"); + + List properList = ResourceUtils.getRequestedResourcesFromConfig(conf, "mapreduce.mapper.proper.rt."); + List incorrectList = ResourceUtils.getRequestedResourcesFromConfig(conf, "mapreduce.mapper.incorrect.rt."); + + Assert.assertEquals(properList.size(), 5); + Assert.assertEquals(incorrectList.size(), 0); + } + @Test public void testGetResourceTypesConfigErrors() throws Exception { Configuration conf = new YarnConfiguration(); -- 2.17.2 (Apple Git-113)