diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 59e108a..f3fcad8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -163,6 +163,11 @@ RM_PREFIX + "scheduler.client.thread-count"; public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; + /** The update interval for calculating resources in FairScheduler .*/ + public static final String RM_SCHEDULER_FS_UPDATE_INTERVAL_MS = + RM_PREFIX + "scheduler.fair.update-interval-ms"; + public static final int DEFAULT_RM_SCHEDULER_FS_UPDATE_INTERVAL_MS = 500; + /** If the port should be included or not in the node name. The node name * is used by the scheduler for resource requests allocation location * matching. Typically this is just the hostname, using the port is needed diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 3a847ce..68c82e1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -135,7 +135,7 @@ public static final Resource CONTAINER_RESERVED = Resources.createResource(-1); // How often fair shares are re-calculated (ms) - protected long UPDATE_INTERVAL = 500; + protected long updateInterval; private final int UPDATE_DEBUG_FREQUENCY = 5; private int updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; @@ -244,13 +244,13 @@ public QueueManager getQueueManager() { /** * A runnable which calls {@link FairScheduler#update()} every - * UPDATE_INTERVAL milliseconds. + * updateInterval milliseconds. */ private class UpdateThread implements Runnable { public void run() { while (true) { try { - Thread.sleep(UPDATE_INTERVAL); + Thread.sleep(updateInterval); update(); preemptTasksIfNecessary(); } catch (Exception e) { @@ -1226,6 +1226,19 @@ private synchronized void initScheduler(Configuration conf) waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill(); usePortForNodeName = this.conf.getUsePortForNodeName(); + updateInterval = conf.getInt( + YarnConfiguration.RM_SCHEDULER_FS_UPDATE_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_FS_UPDATE_INTERVAL_MS + ); + if (updateInterval < 0) { + updateInterval = + YarnConfiguration.DEFAULT_RM_SCHEDULER_FS_UPDATE_INTERVAL_MS; + LOG.warn(YarnConfiguration.RM_SCHEDULER_FS_UPDATE_INTERVAL_MS + + " is invalid, so use default value " + + + YarnConfiguration.DEFAULT_RM_SCHEDULER_FS_UPDATE_INTERVAL_MS + + " ms instead"); + } + rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf); // This stores per-application scheduling information this.applications = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index 310104b..903c7af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -94,7 +94,7 @@ private void startResourceManager(float utilizationThreshold) { scheduler = (FairScheduler)resourceManager.getResourceScheduler(); scheduler.setClock(clock); - scheduler.UPDATE_INTERVAL = 60 * 1000; + scheduler.updateInterval = 60 * 1000; } private void registerNodeAndSubmitApp(