diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index f7c75a6079f..017768c021b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -875,16 +875,6 @@ public static Resource createResourceFromString( resourceValue); } - // special handle gpu - if (key.equals("gpu")) { - key = ResourceInformation.GPU_URI; - } - - // special handle fpga - if (key.equals("fpga")) { - key = ResourceInformation.FPGA_URI; - } - resources.put(key, resourceValue); } return resources; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 8f742fa902f..3a9c6d2ee83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -174,7 +174,7 @@ public void testGetResourceTypesConfigs() throws Exception { new ResourceFileInformation("resource-types-4.xml", 5); testFile4.resourceNameUnitsMap.put("resource1", "G"); testFile4.resourceNameUnitsMap.put("resource2", "m"); - testFile4.resourceNameUnitsMap.put("yarn.io/gpu", ""); + testFile4.resourceNameUnitsMap.put(ResourceInformation.GPU_URI, ""); ResourceFileInformation[] tests = {testFile1, testFile2, testFile3, testFile4}; @@ -201,8 +201,8 @@ public void testGetRequestedResourcesFromConfig() { //these resource type configurations should be recognised String propertyPrefix = "mapreduce.mapper.proper.rt."; String[] expectedKeys = { - "yarn.io/gpu", - "yarn.io/fpga", + ResourceInformation.GPU_URI, + ResourceInformation.FPGA_URI, "yarn.io/anything_without_a_dot", "regular_rt", "regular_rt/with_slash"}; @@ -210,7 +210,7 @@ public void testGetRequestedResourcesFromConfig() { String[] invalidKeys = { propertyPrefix + "too.many_parts", propertyPrefix + "yarn.notio/gpu", - "incorrect.prefix.yarn.io/gpu", + "incorrect.prefix." + ResourceInformation.GPU_URI, propertyPrefix + "yarn.io/", propertyPrefix}; @@ -363,8 +363,8 @@ public void testGetResourceInformation() throws Exception { ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2L)); - test3Resources.setResourceInformation("yarn.io/gpu", - ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + test3Resources.setResourceInformation(ResourceInformation.GPU_URI, + ResourceInformation.newInstance(ResourceInformation.GPU_URI, "", 1)); testRun.put("node-resources-2.xml", test3Resources); for (Map.Entry entry : testRun.entrySet()) { @@ -401,7 +401,7 @@ public void testGetNodeResourcesConfigErrors() throws Exception { public void testGetNodeResourcesRedefineFpgaErrors() throws Exception { Configuration conf = new YarnConfiguration(); expexted.expect(YarnRuntimeException.class); - expexted.expectMessage("Defined mandatory resource type=yarn.io/fpga"); + expexted.expectMessage("Defined mandatory resource type=" + ResourceInformation.FPGA_URI); setupResourceTypesInternal(conf, "resource-types-error-redefine-fpga-unit.xml"); } @@ -410,7 +410,7 @@ public void testGetNodeResourcesRedefineFpgaErrors() throws Exception { public void testGetNodeResourcesRedefineGpuErrors() throws Exception { Configuration conf = new YarnConfiguration(); expexted.expect(YarnRuntimeException.class); - expexted.expectMessage("Defined mandatory resource type=yarn.io/gpu"); + expexted.expectMessage("Defined mandatory resource type=" + ResourceInformation.GPU_URI); setupResourceTypesInternal(conf, "resource-types-error-redefine-gpu-unit.xml"); } @@ -418,7 +418,7 @@ public void testGetNodeResourcesRedefineGpuErrors() throws Exception { @Test public void testResourceNameFormatValidation() { String[] validNames = new String[] { - "yarn.io/gpu", + ResourceInformation.GPU_URI, "gpu", "g_1_2", "123.io/gpu", @@ -467,8 +467,8 @@ public void testGetResourceInformationWithDiffUnits() throws Exception { //2M should be converted to 2000000000m test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "M", 2L)); - test3Resources.setResourceInformation("yarn.io/gpu", - ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + test3Resources.setResourceInformation(ResourceInformation.GPU_URI, + ResourceInformation.newInstance(ResourceInformation.GPU_URI, "", 1)); testRun.put("node-resources-3.xml", test3Resources); for (Map.Entry entry : testRun.entrySet()) { @@ -524,15 +524,6 @@ public void testResourceUnitParsing() throws Exception { ResourceUtils.getResourcesTypeInfo()); resTypes.add(ResourceTypeInfo.newInstance(ResourceInformation.GPU_URI, "")); ResourceUtils.reinitializeResources(resTypes); - res = ResourceUtils.createResourceFromString("memory=2G,vcores=3,gpu=0", - resTypes); - Assert.assertEquals(2 * 1024, res.getMemorySize()); - Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI)); - - res = ResourceUtils.createResourceFromString("memory=2G,vcores=3,gpu=3", - resTypes); - Assert.assertEquals(2 * 1024, res.getMemorySize()); - Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI)); res = ResourceUtils.createResourceFromString("memory=2G,vcores=3", resTypes); @@ -540,12 +531,14 @@ public void testResourceUnitParsing() throws Exception { Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI)); res = ResourceUtils.createResourceFromString( - "memory=2G,vcores=3,yarn.io/gpu=0", resTypes); + "memory=2G,vcores=3," + + ResourceInformation.GPU_URI + "=0", resTypes); Assert.assertEquals(2 * 1024, res.getMemorySize()); Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI)); res = ResourceUtils.createResourceFromString( - "memory=2G,vcores=3,yarn.io/gpu=3", resTypes); + "memory=2G,vcores=3," + + ResourceInformation.GPU_URI + "=3", resTypes); Assert.assertEquals(2 * 1024, res.getMemorySize()); Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI)); } @@ -563,8 +556,8 @@ public void testMultipleOpsForResourcesWithTags() throws Exception { resourceA.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "M", 2L)); - resourceA.setResourceInformation("yarn.io/gpu", - ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + resourceA.setResourceInformation(ResourceInformation.GPU_URI, + ResourceInformation.newInstance(ResourceInformation.GPU_URI, "", 1)); resourceA.setResourceInformation("yarn.io/test-volume", ResourceInformation.newInstance("yarn.io/test-volume", "", 2)); @@ -573,8 +566,8 @@ public void testMultipleOpsForResourcesWithTags() throws Exception { resourceB.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "M", 4L)); - resourceB.setResourceInformation("yarn.io/gpu", - ResourceInformation.newInstance("yarn.io/gpu", "", 2)); + resourceB.setResourceInformation(ResourceInformation.GPU_URI, + ResourceInformation.newInstance(ResourceInformation.GPU_URI, "", 2)); resourceB.setResourceInformation("yarn.io/test-volume", ResourceInformation.newInstance("yarn.io/test-volume", "", 3)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java index 7b0254cdcc5..61307916128 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java @@ -21,6 +21,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; @@ -121,7 +122,7 @@ public void testCapacitySchedulerJobWhenConfigureCustomResourceType() rm.start(); //register node with custom resource - String customResourceType = "yarn.io/gpu"; + String customResourceType = ResourceInformation.GPU_URI; Resource nodeResource = Resources.createResource(4 * g, 4); nodeResource.setResourceValue(customResourceType, 10); MockNM nm1 = rm.registerNode("h1:1234", nodeResource); @@ -186,11 +187,11 @@ public void testCapacitySchedulerInitWithCustomResourceType() Assert.assertNotEquals(0, ResourceUtils .fetchMaximumAllocationFromConfig(cs.getConfiguration()) - .getResourceValue("yarn.io/gpu")); + .getResourceValue(ResourceInformation.GPU_URI)); // Ensure custom resource type exists in queue's maximumAllocation Assert.assertNotEquals(0, cs.getMaximumResourceCapability("a") - .getResourceValue("yarn.io/gpu")); + .getResourceValue(ResourceInformation.GPU_URI)); rm.close(); }