diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index a9c98bd77da..fe64f514692 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -28,9 +28,13 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.File; +import java.io.IOException; +import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -47,13 +51,16 @@ int resourceCount; Map resourceNameUnitsMap; - public ResourceFileInformation(String name, int count) { + ResourceFileInformation(String name, int count) { filename = name; resourceCount = count; resourceNameUnitsMap = new HashMap<>(); } } + @Rule + public ExpectedException expexted = ExpectedException.none(); + @Before public void setup() { ResourceUtils.resetResourceTypes(); @@ -61,10 +68,10 @@ public void setup() { @After public void teardown() { - if(nodeResourcesFile != null && nodeResourcesFile.exists()) { + if (nodeResourcesFile != null && nodeResourcesFile.exists()) { nodeResourcesFile.delete(); } - if(resourceTypesFile != null && resourceTypesFile.exists()) { + if (resourceTypesFile != null && resourceTypesFile.exists()) { resourceTypesFile.delete(); } } @@ -87,8 +94,7 @@ private void testMemoryAndVcores(Map res) { } @Test - public void testGetResourceTypes() throws Exception { - + public void testGetResourceTypes() { Map res = ResourceUtils.getResourceTypes(); Assert.assertEquals(2, res.size()); testMemoryAndVcores(res); @@ -96,7 +102,6 @@ public void testGetResourceTypes() throws Exception { @Test public void testGetResourceTypesConfigs() throws Exception { - Configuration conf = new YarnConfiguration(); ResourceFileInformation testFile1 = @@ -118,16 +123,11 @@ public void testGetResourceTypesConfigs() throws Exception { Map res; for (ResourceFileInformation testInformation : tests) { ResourceUtils.resetResourceTypes(); - File source = new File( - conf.getClassLoader().getResource(testInformation.filename) - .getFile()); - resourceTypesFile = new File(source.getParent(), "resource-types.xml"); - FileUtils.copyFile(source, resourceTypesFile); - res = ResourceUtils.getResourceTypes(); + res = setupResourceTypesInternal(conf, testInformation.filename); testMemoryAndVcores(res); Assert.assertEquals(testInformation.resourceCount, res.size()); - for (Map.Entry entry : testInformation.resourceNameUnitsMap - .entrySet()) { + for (Map.Entry entry : + testInformation.resourceNameUnitsMap.entrySet()) { String resourceName = entry.getKey(); Assert.assertTrue("Missing key " + resourceName, res.containsKey(resourceName)); @@ -137,7 +137,7 @@ public void testGetResourceTypesConfigs() throws Exception { } @Test - public void testGetResourceTypesConfigErrors() throws Exception { + public void testGetResourceTypesConfigErrors() throws IOException { Configuration conf = new YarnConfiguration(); String[] resourceFiles = {"resource-types-error-1.xml", @@ -146,22 +146,16 @@ public void testGetResourceTypesConfigErrors() throws Exception { for (String resourceFile : resourceFiles) { ResourceUtils.resetResourceTypes(); try { - File source = - new File(conf.getClassLoader().getResource(resourceFile).getFile()); - resourceTypesFile = new File(source.getParent(), "resource-types.xml"); - FileUtils.copyFile(source, resourceTypesFile); - ResourceUtils.getResourceTypes(); + setupResourceTypesInternal(conf, resourceFile); Assert.fail("Expected error with file " + resourceFile); - } catch (NullPointerException ne) { - throw ne; - } catch (Exception e) { + } catch (YarnRuntimeException | IllegalArgumentException e) { //Test passed } } } @Test - public void testInitializeResourcesMap() throws Exception { + public void testInitializeResourcesMap() { String[] empty = {"", ""}; String[] res1 = {"resource1", "m"}; String[] res2 = {"resource2", "G"}; @@ -222,8 +216,7 @@ public void testInitializeResourcesMap() throws Exception { } @Test - public void testInitializeResourcesMapErrors() throws Exception { - + public void testInitializeResourcesMapErrors() { String[] mem1 = {"memory-mb", ""}; String[] vcores1 = {"vcores", "M"}; @@ -263,11 +256,9 @@ public void testInitializeResourcesMapErrors() throws Exception { @Test public void testGetResourceInformation() throws Exception { - Configuration conf = new YarnConfiguration(); Map testRun = new HashMap<>(); - setupResourceTypes(conf, "resource-types-4.xml"); - // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); + setupResourceTypesInternal(conf, "resource-types-4.xml"); Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); @@ -280,12 +271,8 @@ public void testGetResourceInformation() throws Exception { for (Map.Entry entry : testRun.entrySet()) { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); - File source = new File( - conf.getClassLoader().getResource(resourceFile).getFile()); - nodeResourcesFile = new File(source.getParent(), "node-resources.xml"); - FileUtils.copyFile(source, nodeResourcesFile); - Map actual = ResourceUtils - .getNodeResourceInformation(conf); + Map actual = setupNodeResources(conf, + resourceFile); Assert.assertEquals(actual.size(), entry.getValue().getResources().length); for (ResourceInformation resInfo : entry.getValue().getResources()) { @@ -297,28 +284,40 @@ public void testGetResourceInformation() throws Exception { @Test public void testGetNodeResourcesConfigErrors() throws Exception { Configuration conf = new YarnConfiguration(); - Map testRun = new HashMap<>(); - setupResourceTypes(conf, "resource-types-4.xml"); - String invalidNodeResFiles[] = { "node-resources-error-1.xml"}; + setupResourceTypesInternal(conf, "resource-types-4.xml"); + String[] invalidNodeResFiles = {"node-resources-error-1.xml"}; for (String resourceFile : invalidNodeResFiles) { ResourceUtils.resetNodeResources(); try { - File source = new File(conf.getClassLoader().getResource(resourceFile).getFile()); - nodeResourcesFile = new File(source.getParent(), "node-resources.xml"); - FileUtils.copyFile(source, nodeResourcesFile); - Map actual = ResourceUtils.getNodeResourceInformation(conf); + setupNodeResources(conf, resourceFile); Assert.fail("Expected error with file " + resourceFile); - } catch (NullPointerException ne) { - throw ne; - } catch (Exception e) { + } catch (YarnRuntimeException e) { //Test passed } } } @Test - public void testResourceNameFormatValidation() throws Exception { + public void testGetNodeResourcesRedefineFpgaErrors() throws Exception { + Configuration conf = new YarnConfiguration(); + expexted.expect(YarnRuntimeException.class); + expexted.expectMessage("Defined mandatory resource type=yarn.io/fpga"); + setupResourceTypesInternal(conf, + "resource-types-error-redefine-fpga-unit.xml"); + } + + @Test + public void testGetNodeResourcesRedefineGpuErrors() throws Exception { + Configuration conf = new YarnConfiguration(); + expexted.expect(YarnRuntimeException.class); + expexted.expectMessage("Defined mandatory resource type=yarn.io/gpu"); + setupResourceTypesInternal(conf, + "resource-types-error-redefine-gpu-unit.xml"); + } + + @Test + public void testResourceNameFormatValidation() { String[] validNames = new String[] { "yarn.io/gpu", "gpu", @@ -355,10 +354,9 @@ public void testResourceNameFormatValidation() throws Exception { @Test public void testGetResourceInformationWithDiffUnits() throws Exception { - Configuration conf = new YarnConfiguration(); Map testRun = new HashMap<>(); - setupResourceTypes(conf, "resource-types-4.xml"); + setupResourceTypesInternal(conf, "resource-types-4.xml"); Resource test3Resources = Resource.newInstance(0, 0); //Resource 'resource1' has been passed as 5T @@ -377,12 +375,8 @@ public void testGetResourceInformationWithDiffUnits() throws Exception { for (Map.Entry entry : testRun.entrySet()) { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); - File source = new File( - conf.getClassLoader().getResource(resourceFile).getFile()); - nodeResourcesFile = new File(source.getParent(), "node-resources.xml"); - FileUtils.copyFile(source, nodeResourcesFile); - Map actual = ResourceUtils - .getNodeResourceInformation(conf); + Map actual = setupNodeResources(conf, + resourceFile); Assert.assertEquals(actual.size(), entry.getValue().getResources().length); for (ResourceInformation resInfo : entry.getValue().getResources()) { @@ -391,6 +385,19 @@ public void testGetResourceInformationWithDiffUnits() throws Exception { } } + /** + * This method is dangerous to use as ResourceUtils.getResourceTypes can + * throw an Exception easily and dest.getAbsolutePath() won't be returned + * to the caller. The caller is unable to call a potential cleanup of the + * returned file, leaving remainders of obsolete contents of + * the resource-types.xml file to the next testcase that potentionally tests + * resource types configuration. + * @param conf + * @param filename + * @return + * @throws Exception + */ + @Deprecated public static String setupResourceTypes(Configuration conf, String filename) throws Exception { File source = new File( @@ -400,4 +407,33 @@ public static String setupResourceTypes(Configuration conf, String filename) ResourceUtils.getResourceTypes(); return dest.getAbsolutePath(); } + + private Map setupResourceTypesInternal( + Configuration conf, String srcFileName) throws IOException { + URL srcFileUrl = conf.getClassLoader().getResource(srcFileName); + if (srcFileUrl == null) { + throw new IllegalArgumentException( + "Source file does not exist: " + srcFileName); + } + File source = new File(srcFileUrl.getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + this.resourceTypesFile = dest; + return ResourceUtils.getResourceTypes(); + } + + private Map setupNodeResources( + Configuration conf, String srcFileName) throws IOException { + URL srcFileUrl = conf.getClassLoader().getResource(srcFileName); + if (srcFileUrl == null) { + throw new IllegalArgumentException( + "Source file does not exist: " + srcFileName); + } + File source = new File(srcFileUrl.getFile()); + File dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + this.nodeResourcesFile = dest; + return ResourceUtils + .getNodeResourceInformation(conf); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml new file mode 100644 index 00000000000..72cfb9888a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml @@ -0,0 +1,45 @@ + + + + + + + + + + yarn.resource-types + yarn.io/fpga + + + + yarn.resource-types.yarn.io/fpga.units + G + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml new file mode 100644 index 00000000000..aa61b5f40bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml @@ -0,0 +1,45 @@ + + + + + + + + + + yarn.resource-types + yarn.io/gpu + + + + yarn.resource-types.yarn.io/gpu.units + G + + +