diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java index 3c3c2cce2d1..d39a28b49a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java @@ -37,10 +37,10 @@ private static final Pattern RESOURCE_VALUE_AND_UNIT_PATTERN = Pattern.compile("(\\d+)([A-za-z]*)"); - private ResourceTypesTestHelper() {} + private ResourceTypesTestHelper() { } private static final RecordFactory RECORD_FACTORY = RecordFactoryProvider - .getRecordFactory(null); + .getRecordFactory(null); private static final class ResourceValueAndUnit { private final Long value; @@ -53,39 +53,40 @@ private ResourceValueAndUnit(Long value, String unit) { } public static Resource newResource(long memory, int vCores, Map customResources) { + String> customResources) { Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class); resource.setMemorySize(memory); resource.setVirtualCores(vCores); - - for (Map.Entry customResource : - customResources.entrySet()) { - String resourceName = customResource.getKey(); - ResourceInformation resourceInformation = - createResourceInformation(resourceName, - customResource.getValue()); - resource.setResourceInformation(resourceName, resourceInformation); + if (customResources != null) { + for (Map.Entry customResource : + customResources.entrySet()) { + String resourceName = customResource.getKey(); + ResourceInformation resourceInformation = + createResourceInformation(resourceName, + customResource.getValue()); + resource.setResourceInformation(resourceName, resourceInformation); + } } return resource; } - public static ResourceInformation createResourceInformation(String - resourceName, String descriptor) { + public static ResourceInformation createResourceInformation( + String resourceName, String descriptor) { ResourceValueAndUnit resourceValueAndUnit = - getResourceValueAndUnit(descriptor); + getResourceValueAndUnit(descriptor); return ResourceInformation - .newInstance(resourceName, resourceValueAndUnit.unit, - resourceValueAndUnit.value); + .newInstance(resourceName, resourceValueAndUnit.unit, + resourceValueAndUnit.value); } private static ResourceValueAndUnit getResourceValueAndUnit(String val) { Matcher matcher = RESOURCE_VALUE_AND_UNIT_PATTERN.matcher(val); if (!matcher.find()) { throw new RuntimeException("Invalid pattern of resource descriptor: " + - val); + val); } else if (matcher.groupCount() != 2) { throw new RuntimeException("Capturing group count in string " + - val + " is not 2!"); + val + " is not 2!"); } long value = Long.parseLong(matcher.group(1)); @@ -107,7 +108,12 @@ private static ResourceValueAndUnit getResourceValueAndUnit(String val) { public static Map extractCustomResourcesAsStrings( Resource res) { Map resValues = extractCustomResources(res); - return resValues.entrySet().stream() + return convertCustomResources(resValues); + } + + public static Map convertCustomResources( + Map customResources) { + return customResources.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, e -> String.valueOf(e.getValue()))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java index f13ae4ac06b..9ac793fa2b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java @@ -20,6 +20,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -112,8 +112,11 @@ protected abstract Resource getResourceUsageForQueue(ResourceManager rm, private void requestResources(MockAM am, long memory, int vCores, Map customResources) throws Exception { + Map convertedCustomResources = + ResourceTypesTestHelper.convertCustomResources(customResources); am.allocate(Collections.singletonList(ResourceRequest.newBuilder() - .capability(TestUtils.createResource(memory, vCores, customResources)) + .capability(ResourceTypesTestHelper.newResource( + memory, vCores, convertedCustomResources)) .numContainers(1) .resourceName("*") .build()), null); @@ -551,8 +554,8 @@ public void testValidateRequestCapacityAgainstMinMaxAllocation() MockRM rm = new MockRM(yarnConf); rm.start(); - MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, + ResourceTypesTestHelper.newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null)); MockRMAppSubmissionData data = @@ -615,10 +618,10 @@ public void testRequestCapacityMinMaxAllocationForResourceTypes() MockRM rm = new MockRM(yarnConf); rm.start(); - MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, + ResourceTypesTestHelper.newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - ImmutableMap.of(CUSTOM_RES, 4))); + ImmutableMap.of(CUSTOM_RES, "4"))); MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index 182016a6cfa..89cc79cf727 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -60,7 +60,7 @@ @Override protected YarnConfiguration createYarnConfig() { CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + new CapacitySchedulerConfiguration(); csConf.setResourceComparator(DominantResourceCalculator.class); YarnConfiguration yarnConf = new YarnConfiguration(csConf); yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, @@ -69,8 +69,7 @@ protected YarnConfiguration createYarnConfig() { } @Override - protected Resource getResourceUsageForQueue(ResourceManager rm, - String queue) { + protected Resource getResourceUsageForQueue(ResourceManager rm, String queue) { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue(DEFAULT_QUEUE); return leafQueue.getUsedResources(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 47606925af8..62b43e8e3bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -298,16 +299,16 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { MockRM rm = new MockRM(conf); rm.start(); - Map nameToValues = new HashMap<>(); - nameToValues.put(ResourceInformation.GPU_URI, 4); + Map nameToValues = new HashMap<>(); + nameToValues.put(ResourceInformation.GPU_URI, "4"); // Register NM1 with 10GB memory, 4 CPU and 4 GPU MockNM nm1 = rm.registerNode("127.0.0.1:1234", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); nameToValues.clear(); // Register NM2 with 10GB memory, 4 CPU and 0 GPU rm.registerNode("127.0.0.1:1235", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); RMApp app1 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(1024, rm) @@ -330,9 +331,9 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { Assert.assertEquals(4, report_nm1.getAvailableResource() .getResourceInformation(ResourceInformation.GPU_URI).getValue()); - nameToValues.put(ResourceInformation.GPU_URI, 4); + nameToValues.put(ResourceInformation.GPU_URI, "4"); Resource containerGpuResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one container which takes all 4 GPU am1.allocate( @@ -355,7 +356,7 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { nameToValues.clear(); Resource containerResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one more container which doesnt need GPU am1.allocate( Collections.singletonList(ResourceRequest.newInstance( @@ -479,8 +480,8 @@ public void testConsumeAllExtendedResourcesWithSmallMinUserLimitPct() for (int i = 0; i < 8; i++) { fiCaApp1.updateResourceRequests(Collections.singletonList( ResourceRequest.newBuilder() - .capability(TestUtils.createResource(1 * GB, 1, - ImmutableMap.of("res_1", 10))) + .capability(ResourceTypesTestHelper.newResource(1 * GB, 1, + ImmutableMap.of("res_1", "10"))) .numContainers(1) .resourceName("*") .build())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 15030a768f9..44b445156dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -467,23 +467,4 @@ public FiCaSchedulerApp getApplicationAttempt( cs.submitResourceCommitRequest(clusterResource, csAssignment); } - - /** - * An easy way to create resources other than memory and vcores for tests. - * @param memory memory - * @param vcores vcores - * @param nameToValues resource types other than memory and vcores. - * @return created resource - */ - public static Resource createResource(long memory, int vcores, - Map nameToValues) { - Resource res = Resource.newInstance(memory, vcores); - if (nameToValues != null) { - for (Map.Entry entry : nameToValues.entrySet()) { - res.setResourceInformation(entry.getKey(), ResourceInformation - .newInstance(entry.getKey(), "", entry.getValue())); - } - } - return res; - } }