diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index e5d2a066c4a..5056e038db6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -149,11 +149,6 @@ private static final Log STATE_DUMP_LOG = LogFactory.getLog(FairScheduler.class.getName() + ".statedump"); - private static final ResourceCalculator RESOURCE_CALCULATOR = - new DefaultResourceCalculator(); - private static final ResourceCalculator DOMINANT_RESOURCE_CALCULATOR = - new DominantResourceCalculator(); - // Value that container assignment methods return when a container is // reserved public static final Resource CONTAINER_RESERVED = Resources.createResource(-1); @@ -208,6 +203,8 @@ @VisibleForTesting Resource reservationThreshold; + private ResourceCalculator resourceCalculator; + public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); @@ -846,7 +843,7 @@ private void removeNode(RMNode rmNode) { public Resource getNormalizedResource(Resource requestedResource, Resource maxResourceCapability) { return SchedulerUtils.getNormalizedResource(requestedResource, - DOMINANT_RESOURCE_CALCULATOR, + resourceCalculator, minimumAllocation, maxResourceCapability, incrAllocation); @@ -1079,7 +1076,7 @@ void continuousSchedulingAttempt() throws InterruptedException { @Override public int compare(FSSchedulerNode n1, FSSchedulerNode n2) { - return RESOURCE_CALCULATOR.compare(getClusterResource(), + return resourceCalculator.compare(getClusterResource(), n2.getUnallocatedResource(), n1.getUnallocatedResource()); } @@ -1193,7 +1190,7 @@ public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { @Override public ResourceCalculator getResourceCalculator() { - return RESOURCE_CALCULATOR; + return resourceCalculator; } /** @@ -1451,6 +1448,7 @@ private void initScheduler(Configuration conf) throws IOException { } catch (Exception e) { throw new IOException("Failed to start FairScheduler", e); } + initResourceCalculator(); if (continuousSchedulingEnabled) { // Continuous scheduling is deprecated log it on startup @@ -1484,6 +1482,16 @@ private void initScheduler(Configuration conf) throws IOException { } } + private void initResourceCalculator() { + final int numResources = + ResourceUtils.getNumberOfKnownResourceTypes(); + if (numResources > 2) { + this.resourceCalculator = new DominantResourceCalculator(); + } else { + this.resourceCalculator = new DefaultResourceCalculator(); + } + } + @VisibleForTesting protected void createPreemptionThread() { preemptionThread = new FSPreemptionThread(this); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..83c7fc2d53e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -118,8 +120,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -130,6 +135,7 @@ import org.xml.sax.SAXException; import com.google.common.collect.Sets; +import sun.jvm.hotspot.debugger.cdbg.BaseClass; @SuppressWarnings("unchecked") public class TestFairScheduler extends FairSchedulerTestBase { @@ -165,8 +171,36 @@ public void tearDown() { QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); YarnAuthorizationProvider.destroy(); + CustomResourceTypesConfigurationProvider.reset(); } + private void initResourceTypes(Configuration conf) { + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + + + @Test + public void testResourceCalculatorIsDefaultIfCustomResourceIsNotDefined() { + scheduler.init(conf); + scheduler.start(); + + assertThat(scheduler.getResourceCalculator(), + instanceOf(DefaultResourceCalculator.class)); + } + + @Test + public void testResourceCalculatorIsDominantIfCustomResourceIsDefined() { + initResourceTypes(conf); + CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(3); + + scheduler.init(conf); + scheduler.start(); + + assertThat(scheduler.getResourceCalculator(), + instanceOf(DominantResourceCalculator.class)); + } @Test (timeout = 30000) public void testConfValidation() throws Exception {