diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 618ee20cfe0..c215352d5a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -341,6 +342,7 @@ void initScheduler(Configuration configuration) throws } this.csConfProvider.init(configuration); this.conf = this.csConfProvider.loadConfiguration(configuration); + loadResourceTypesConfiguration(this.conf); validateConf(this.conf); this.minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); @@ -417,6 +419,25 @@ void initScheduler(Configuration configuration) throws } } + /** + * RM init will not load resource-types.xml into configuration. + * Which will cause application fail to submit with custom resource. + * see details in YARN-9205. + * */ + protected void loadResourceTypesConfiguration(Configuration conf) throws IOException { + try { + InputStream typeInputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + if (typeInputStream != null) { + conf.addResource(typeInputStream); + } + } catch (Exception e) { + throw new IOException(e); + } + } + private void startSchedulerThreads() { try { writeLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index d6ffd7763ad..dcf39f3b0d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -205,6 +205,17 @@ public AllocateResponse allocate( labelExpression, -1); return allocate(reqs, releases); } + + public AllocateResponse allocate( + String host, Resource cap, int numContainers, + List releases, String labelExpression) throws Exception { + List reqs = new ArrayList<>(); + ResourceRequest oneReq = + createResourceReq(host, cap, numContainers, + labelExpression); + reqs.add(oneReq); + return allocate(reqs, releases); + } public List createReq(String[] hosts, int memory, int priority, int containers, long allocationRequestId) throws Exception { @@ -272,6 +283,22 @@ public ResourceRequest createResourceReq(String resource, int memory, } + public ResourceRequest createResourceReq(String host, Resource cap, + int containers, String labelExpression) throws Exception { + ResourceRequest req = Records.newRecord(ResourceRequest.class); + req.setResourceName(host); + req.setNumContainers(containers); + Priority pri = Records.newRecord(Priority.class); + pri.setPriority(1); + req.setPriority(pri); + req.setCapability(cap); + if (labelExpression != null) { + req.setNodeLabelExpression(labelExpression); + } + req.setExecutionTypeRequest(ExecutionTypeRequest.newInstance()); + return req; + + } public AllocateResponse allocate( List resourceRequest, List releases) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 9f01a172891..cf69b403c39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -18,15 +18,20 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.SecurityUtilTestHelper; +import org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -44,10 +49,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -64,13 +71,20 @@ import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.util.resource.TestResourceUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestContainerAllocation { @@ -1187,4 +1201,116 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() rm1.close(); } + + /** + * Test containers request custom resource. + * */ + @Test + public void testCapacitySchedulerJobWhenConfigureCustomResourceType() + throws Exception { + // reset resource types + ResourceUtils.resetResourceTypes(); + String resourceTypesFile = "resource-types-test.xml"; + File source = new File( + conf.getClassLoader().getResource(resourceTypesFile).getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + + CapacitySchedulerConfiguration newConf = + (CapacitySchedulerConfiguration) TestUtils + .getConfigurationWithMultipleQueues(conf); + newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + + MAXIMUM_ALLOCATION_MB, "4096"); + //start RM + MockRM rm = new MockRM(newConf); + rm.start(); + + //register node with custom resource + String customResourceType = "yarn.io/gpu"; + Resource nodeResource = Resources.createResource(4 * GB, 4); + nodeResource.setResourceValue(customResourceType, 10); + MockNM nm1 = rm.registerNode("h1:1234", nodeResource); + + // submit app + Resource amResource = Resources.createResource(1 * GB, 1); + amResource.setResourceValue(customResourceType, 1); + RMApp app1 = rm.submitApp(amResource, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + // am request containers + Resource cResource = Resources.createResource(1 * GB, 1); + amResource.setResourceValue(customResourceType, 1); + am1.allocate("*", cResource, 2, + new ArrayList(), null); + + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); + FiCaSchedulerApp schedulerApp1 = + cs.getApplicationAttempt(am1.getApplicationAttemptId()); + + // Do nm heartbeats 1 times, will allocate a container on nm1 + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + rm.drainEvents(); + Assert.assertEquals(2, schedulerApp1.getLiveContainers().size()); + rm.close(); + + // cleanup resource-types.xml + dest.delete(); + } + + /** + * Test CS initialized with custom resource types loaded. + * */ + @Test + public void testCapacitySchedulerInitWithCustomResourceType() + throws IOException { + Configuration conf = new YarnConfiguration(); + // reset resource types + ResourceUtils.resetResourceTypes(); + String resourceTypesFile = "resource-types-test.xml"; + File source = new File( + conf.getClassLoader().getResource(resourceTypesFile).getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + + CapacityScheduler cs = new CapacityScheduler(); + CapacityScheduler spyCS = spy(cs); + CapacitySchedulerConfiguration csConf = + (CapacitySchedulerConfiguration) TestUtils + .getConfigurationWithMultipleQueues(conf);; + csConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + spyCS.setConf(csConf); + + RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); + nodeLabelsManager.init(csConf); + PlacementManager pm = new PlacementManager(); + RMContext mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); + mockContext.setNodeLabelManager(nodeLabelsManager); + when(mockContext.getNodeLabelManager()).thenReturn(nodeLabelsManager); + when(mockContext.getQueuePlacementManager()).thenReturn(pm); + spyCS.setRMContext(mockContext); + + spyCS.init(csConf); + + // Ensure invoke method to load the resource-types.xml + verify(spyCS).loadResourceTypesConfiguration( + any(CapacitySchedulerConfiguration.class)); + // Ensure the method can get custom resource type from + // CapacitySchedulerConfiguration + Assert.assertNotEquals(0, + ResourceUtils.fetchMaximumAllocationFromConfig(spyCS.getConfiguration()) + .getResourceValue("yarn.io/gpu")); + // Ensure custom resource type exists in queue's maximumAllocation + Assert.assertNotEquals(0, + spyCS.getMaximumResourceCapability("a") + .getResourceValue("yarn.io/gpu")); + // cleanup resource-types.xml + dest.delete(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml new file mode 100644 index 00000000000..571fae32f1e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml @@ -0,0 +1,23 @@ + + + + + + + + yarn.resource-types + yarn.io/gpu + + \ No newline at end of file