diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b2978d4..d9be824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -388,18 +388,44 @@ public double getRackLocalityThreshold() { return rackLocalityThreshold; } + /** + * Delay in milliseconds for locality fallback node to rack. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return delay in ms + */ + @Deprecated public long getNodeLocalityDelayMs() { return nodeLocalityDelayMs; } + /** + * Delay in milliseconds for locality fallback rack to other. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return delay in ms + */ + @Deprecated public long getRackLocalityDelayMs() { return rackLocalityDelayMs; } + /** + * Whether continuous scheduling is turned on. + * @deprecated Continuous scheduling should not be turned ON. It is + * deprecated because it can cause scheduler slowness due to locking issues. + * Schedulers should use assignmultiple as a replacement. + * @return whether continuous scheduling is enabled + */ + @Deprecated public boolean isContinuousSchedulingEnabled() { return continuousSchedulingEnabled; } + /** + * The sleep time of the continuous scheduler thread. + * @deprecated linked to {@link #isContinuousSchedulingEnabled} deprecation + * @return sleep time in ms + */ + @Deprecated public int getContinuousSchedulingSleepMs() { return continuousSchedulingSleepMs; } @@ -1295,6 +1321,10 @@ private void initScheduler(Configuration conf) throws IOException { } if (continuousSchedulingEnabled) { + // Contiuous scheduling is deprecated log it on startup + LOG.warn("Continuous scheduling is turned ON. It is deprecated " + + "because it can cause scheduler slowness due to locking issues. " + + "Schedulers should use assignmultiple as a replacement."); // start continuous scheduling thread schedulingThread = new ContinuousSchedulingThread(); schedulingThread.setName("FairSchedulerContinuousScheduling"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 9c9eee6..ecd8deb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -92,20 +92,38 @@ protected static final float DEFAULT_LOCALITY_THRESHOLD_RACK = DEFAULT_LOCALITY_THRESHOLD; - /** Delay for node locality. */ - protected static final String LOCALITY_DELAY_NODE_MS = CONF_PREFIX + "locality-delay-node-ms"; + /** + * Delay for node locality. + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String LOCALITY_DELAY_NODE_MS = CONF_PREFIX + + "locality-delay-node-ms"; protected static final long DEFAULT_LOCALITY_DELAY_NODE_MS = -1L; - /** Delay for rack locality. */ - protected static final String LOCALITY_DELAY_RACK_MS = CONF_PREFIX + "locality-delay-rack-ms"; + /** + * Delay for rack locality. + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String LOCALITY_DELAY_RACK_MS = CONF_PREFIX + + "locality-delay-rack-ms"; protected static final long DEFAULT_LOCALITY_DELAY_RACK_MS = -1L; - /** Enable continuous scheduling or not. */ - protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + "continuous-scheduling-enabled"; + /** + * Enable continuous scheduling or not. + * @deprecated Continuous scheduling is known to cause locking issue inside + * the scheduler in larger cluster, more than 100 nodes, use + * {@link #ASSIGN_MULTIPLE} to improve container allocation ramp up. + */ + protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + + "continuous-scheduling-enabled"; protected static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; - /** Sleep time of each pass in continuous scheduling (5ms in default) */ - protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + "continuous-scheduling-sleep-ms"; + /** + * Sleep time of each pass in continuous scheduling (5ms in default). + * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled + */ + protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + + "continuous-scheduling-sleep-ms"; protected static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; /** Whether preemption is enabled. */ @@ -215,18 +233,45 @@ public float getLocalityThresholdRack() { return getFloat(LOCALITY_THRESHOLD_RACK, DEFAULT_LOCALITY_THRESHOLD_RACK); } + /** + * Whether continuous scheduling is turned on. + * @deprecated use {@link #ASSIGN_MULTIPLE} to improve container allocation + * ramp up. + * @return whether continuous scheduling is enabled + */ + @Deprecated public boolean isContinuousSchedulingEnabled() { - return getBoolean(CONTINUOUS_SCHEDULING_ENABLED, DEFAULT_CONTINUOUS_SCHEDULING_ENABLED); + return getBoolean(CONTINUOUS_SCHEDULING_ENABLED, + DEFAULT_CONTINUOUS_SCHEDULING_ENABLED); } + /** + * The sleep time of the continuous scheduler thread. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return sleep time in ms + */ + @Deprecated public int getContinuousSchedulingSleepMs() { - return getInt(CONTINUOUS_SCHEDULING_SLEEP_MS, DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); + return getInt(CONTINUOUS_SCHEDULING_SLEEP_MS, + DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); } + /** + * Delay in milliseconds for locality fallback node to rack. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return delay in ms + */ + @Deprecated public long getLocalityDelayNodeMs() { return getLong(LOCALITY_DELAY_NODE_MS, DEFAULT_LOCALITY_DELAY_NODE_MS); } + /** + * Delay in milliseconds for locality fallback rack to other. + * @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation + * @return delay in ms + */ + @Deprecated public long getLocalityDelayRackMs() { return getLong(LOCALITY_DELAY_RACK_MS, DEFAULT_LOCALITY_DELAY_RACK_MS); }