diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index e5d2a066c4a..5056e038db6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -149,11 +149,6 @@ private static final Log STATE_DUMP_LOG = LogFactory.getLog(FairScheduler.class.getName() + ".statedump"); - private static final ResourceCalculator RESOURCE_CALCULATOR = - new DefaultResourceCalculator(); - private static final ResourceCalculator DOMINANT_RESOURCE_CALCULATOR = - new DominantResourceCalculator(); - // Value that container assignment methods return when a container is // reserved public static final Resource CONTAINER_RESERVED = Resources.createResource(-1); @@ -208,6 +203,8 @@ @VisibleForTesting Resource reservationThreshold; + private ResourceCalculator resourceCalculator; + public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); @@ -846,7 +843,7 @@ private void removeNode(RMNode rmNode) { public Resource getNormalizedResource(Resource requestedResource, Resource maxResourceCapability) { return SchedulerUtils.getNormalizedResource(requestedResource, - DOMINANT_RESOURCE_CALCULATOR, + resourceCalculator, minimumAllocation, maxResourceCapability, incrAllocation); @@ -1079,7 +1076,7 @@ void continuousSchedulingAttempt() throws InterruptedException { @Override public int compare(FSSchedulerNode n1, FSSchedulerNode n2) { - return RESOURCE_CALCULATOR.compare(getClusterResource(), + return resourceCalculator.compare(getClusterResource(), n2.getUnallocatedResource(), n1.getUnallocatedResource()); } @@ -1193,7 +1190,7 @@ public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { @Override public ResourceCalculator getResourceCalculator() { - return RESOURCE_CALCULATOR; + return resourceCalculator; } /** @@ -1451,6 +1448,7 @@ private void initScheduler(Configuration conf) throws IOException { } catch (Exception e) { throw new IOException("Failed to start FairScheduler", e); } + initResourceCalculator(); if (continuousSchedulingEnabled) { // Continuous scheduling is deprecated log it on startup @@ -1484,6 +1482,16 @@ private void initScheduler(Configuration conf) throws IOException { } } + private void initResourceCalculator() { + final int numResources = + ResourceUtils.getNumberOfKnownResourceTypes(); + if (numResources > 2) { + this.resourceCalculator = new DominantResourceCalculator(); + } else { + this.resourceCalculator = new DefaultResourceCalculator(); + } + } + @VisibleForTesting protected void createPreemptionThread() { preemptionThread = new FSPreemptionThread(this); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..2fd6c69570b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -118,15 +119,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.util.resource.TestResourceUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.matchers.InstanceOf; import org.xml.sax.SAXException; import com.google.common.collect.Sets; @@ -165,8 +171,29 @@ public void tearDown() { QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); YarnAuthorizationProvider.destroy(); + CustomResourceTypesConfigurationProvider.reset(); } + @Test + public void testResourceCalculatorIsDefaultIfCustomResourceIsNotDefined() { + scheduler.init(conf); + scheduler.start(); + + assertThat(scheduler.getResourceCalculator(), + new InstanceOf(DefaultResourceCalculator.class)); + } + + @Test + public void testResourceCalculatorIsDominantIfCustomResourceIsDefined() { + TestResourceUtils.addNewTypesToResources("customResource1", + "customResource2", "customResource3"); + + scheduler.init(conf); + scheduler.start(); + + assertThat(scheduler.getResourceCalculator(), + new InstanceOf(DominantResourceCalculator.class)); + } @Test (timeout = 30000) public void testConfValidation() throws Exception {