diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 94991eb..22e5e4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -929,6 +929,7 @@ private boolean isOverAMShareLimit() { return false; } + @SuppressWarnings("deprecation") private Resource assignContainer(FSSchedulerNode node, boolean reserved) { if (LOG.isTraceEnabled()) { LOG.trace("Node offered to app: " + getName() + " reserved: " + reserved); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java index b3e59c5..8ce97c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java @@ -46,6 +46,7 @@ private final Timer preemptionTimer; private final Lock schedulerReadLock; + @SuppressWarnings("deprecation") FSPreemptionThread(FairScheduler scheduler) { setDaemon(true); setName("FSPreemptionThread"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b2978d4..d9be824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -388,18 +388,44 @@ public double getRackLocalityThreshold() { return rackLocalityThreshold; } + /** + * Delay in milliseconds for locality fallback node to rack. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return delay in ms + */ + @Deprecated public long getNodeLocalityDelayMs() { return nodeLocalityDelayMs; } + /** + * Delay in milliseconds for locality fallback rack to other. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return delay in ms + */ + @Deprecated public long getRackLocalityDelayMs() { return rackLocalityDelayMs; } + /** + * Whether continuous scheduling is turned on. + * @deprecated Continuous scheduling should not be turned ON. It is + * deprecated because it can cause scheduler slowness due to locking issues. + * Schedulers should use assignmultiple as a replacement. + * @return whether continuous scheduling is enabled + */ + @Deprecated public boolean isContinuousSchedulingEnabled() { return continuousSchedulingEnabled; } + /** + * The sleep time of the continuous scheduler thread. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return sleep time in ms + */ + @Deprecated public int getContinuousSchedulingSleepMs() { return continuousSchedulingSleepMs; } @@ -1295,6 +1321,10 @@ private void initScheduler(Configuration conf) throws IOException { } if (continuousSchedulingEnabled) { + // Contiuous scheduling is deprecated log it on startup + LOG.warn("Continuous scheduling is turned ON. It is deprecated " + + "because it can cause scheduler slowness due to locking issues. " + + "Schedulers should use assignmultiple as a replacement."); // start continuous scheduling thread schedulingThread = new ContinuousSchedulingThread(); schedulingThread.setName("FairSchedulerContinuousScheduling"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 9c9eee6..ecd8deb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -92,20 +92,38 @@ protected static final float DEFAULT_LOCALITY_THRESHOLD_RACK = DEFAULT_LOCALITY_THRESHOLD; - /** Delay for node locality. */ - protected static final String LOCALITY_DELAY_NODE_MS = CONF_PREFIX + "locality-delay-node-ms"; + /** + * Delay for node locality. + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String LOCALITY_DELAY_NODE_MS = CONF_PREFIX + + "locality-delay-node-ms"; protected static final long DEFAULT_LOCALITY_DELAY_NODE_MS = -1L; - /** Delay for rack locality. */ - protected static final String LOCALITY_DELAY_RACK_MS = CONF_PREFIX + "locality-delay-rack-ms"; + /** + * Delay for rack locality. + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String LOCALITY_DELAY_RACK_MS = CONF_PREFIX + + "locality-delay-rack-ms"; protected static final long DEFAULT_LOCALITY_DELAY_RACK_MS = -1L; - /** Enable continuous scheduling or not. */ - protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + "continuous-scheduling-enabled"; + /** + * Enable continuous scheduling or not. + * @deprecated Continuous scheduling is known to cause locking issue inside + * the scheduler in larger cluster, more than 100 nodes, use + * {@link #ASSIGN_MULTIPLE} to improve container allocation ramp up. + */ + protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + + "continuous-scheduling-enabled"; protected static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; - /** Sleep time of each pass in continuous scheduling (5ms in default) */ - protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + "continuous-scheduling-sleep-ms"; + /** + * Sleep time of each pass in continuous scheduling (5ms in default). + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + + "continuous-scheduling-sleep-ms"; protected static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; /** Whether preemption is enabled. */ @@ -215,18 +233,45 @@ public float getLocalityThresholdRack() { return getFloat(LOCALITY_THRESHOLD_RACK, DEFAULT_LOCALITY_THRESHOLD_RACK); } + /** + * Whether continuous scheduling is turned on. + * @deprecated use {@link #ASSIGN_MULTIPLE} to improve container allocation + * ramp up. + * @return whether continuous scheduling is enabled + */ + @Deprecated public boolean isContinuousSchedulingEnabled() { - return getBoolean(CONTINUOUS_SCHEDULING_ENABLED, DEFAULT_CONTINUOUS_SCHEDULING_ENABLED); + return getBoolean(CONTINUOUS_SCHEDULING_ENABLED, + DEFAULT_CONTINUOUS_SCHEDULING_ENABLED); } + /** + * The sleep time of the continuous scheduler thread. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return sleep time in ms + */ + @Deprecated public int getContinuousSchedulingSleepMs() { - return getInt(CONTINUOUS_SCHEDULING_SLEEP_MS, DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); + return getInt(CONTINUOUS_SCHEDULING_SLEEP_MS, + DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); } + /** + * Delay in milliseconds for locality fallback node to rack. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return delay in ms + */ + @Deprecated public long getLocalityDelayNodeMs() { return getLong(LOCALITY_DELAY_NODE_MS, DEFAULT_LOCALITY_DELAY_NODE_MS); } + /** + * Delay in milliseconds for locality fallback rack to other. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return delay in ms + */ + @Deprecated public long getLocalityDelayRackMs() { return getLong(LOCALITY_DELAY_RACK_MS, DEFAULT_LOCALITY_DELAY_RACK_MS); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java index 854a65c..681123d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java @@ -65,6 +65,7 @@ private ControlledClock mockClock; private static int delayThresholdTimeMs = 1000; + @SuppressWarnings("deprecation") @Override public Configuration createConfiguration() { Configuration conf = super.createConfiguration(); @@ -77,6 +78,7 @@ public Configuration createConfiguration() { return conf; } + @SuppressWarnings("deprecation") @Before public void setup() { mockClock = new ControlledClock(); @@ -187,6 +189,7 @@ public void testSortedNodes() throws Exception { Assert.assertEquals(2, nodes.size()); } + @SuppressWarnings("deprecation") @Test public void testWithNodeRemoved() throws Exception { // Disable continuous scheduling, will invoke continuous @@ -240,6 +243,7 @@ public void testWithNodeRemoved() throws Exception { } } + @SuppressWarnings("deprecation") @Test public void testInterruptedException() throws Exception { @@ -384,6 +388,7 @@ public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { assertEquals(delayThresholdTimeMs, initSchedulerTime); } + @SuppressWarnings("deprecation") private void triggerSchedulingAttempt() throws InterruptedException { Thread.sleep( 2 * scheduler.getConf().getContinuousSchedulingSleepMs()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index cd0570a..6ee2191 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -193,6 +193,7 @@ public void testConfValidation() throws Exception { // TESTS + @SuppressWarnings("deprecation") @Test(timeout=2000) public void testLoadConfigurationOnInitialize() throws IOException { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); @@ -4409,6 +4410,7 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { verifyQueueNumRunnable("queue1.sub3", 0, 0); } + @SuppressWarnings("deprecation") @Test public void testSchedulingOnRemovedNode() throws Exception { // Disable continuous scheduling, will invoke continuous scheduling manually