diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 59e108a..a38053b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -325,6 +325,10 @@
public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED =
false;
+ public static final String RM_SCHEDULER_RECOVERY_WAIT_MS = RM_PREFIX
+ + "scheduler.recovery-wait-ms";
+ public static final long DEFAULT_RM_SCHEDULER_RECOVERY_WAIT_MS = 10000;
+
/** Zookeeper interaction configs */
public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0c1628e..34f1bb8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -278,6 +278,16 @@
+ Set the amount of time scheduler waits before allocating new
+ containers on work-preserving-recovery. Such wait period gives RM a chance
+ to settle down resyncing with NMs in the cluster on recovery before assigning
+ new containers to applications.
+
+ yarn.resourcemanager.scheduler.recovery-wait-ms
+ 10000
+
+
+
The class to use as the persistent store.
If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 517e680..359b99e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -101,4 +101,6 @@ void setRMApplicationHistoryWriter(
ConfigurationProvider getConfigurationProvider();
boolean isWorkPreservingRecoveryEnabled();
+
+ boolean isSchedulerReadyForAllocatingContainers();
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 1abc660..548b4a8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -82,6 +82,8 @@
private ApplicationMasterService applicationMasterService;
private RMApplicationHistoryWriter rmApplicationHistoryWriter;
private ConfigurationProvider configurationProvider;
+ private long schedulerRecoveryStartTime = 0;
+ private long schedulerRecoveryWaitTime = 0;
/**
* Default constructor. To be used in conjunction with setter methods for
@@ -359,4 +361,14 @@ public void setConfigurationProvider(
ConfigurationProvider configurationProvider) {
this.configurationProvider = configurationProvider;
}
+
+ public void setSchedulerRecoveryStartAndWaitTime(long waitTime) {
+ this.schedulerRecoveryStartTime = System.currentTimeMillis();
+ this.schedulerRecoveryWaitTime = waitTime;
+ }
+
+ public boolean isSchedulerReadyForAllocatingContainers() {
+ return (System.currentTimeMillis() - schedulerRecoveryStartTime)
+ > schedulerRecoveryWaitTime;
+ }
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 724dee1..d206261 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1027,6 +1027,8 @@ public void recover(RMState state) throws Exception {
// recover applications
rmAppManager.recover(state);
+
+ setSchedulerRecoveryStartAndWaitTime(state, conf);
}
public static void main(String argv[]) {
@@ -1066,6 +1068,16 @@ private void resetDispatcher() {
rmContext.setDispatcher(rmDispatcher);
}
+ private void setSchedulerRecoveryStartAndWaitTime(RMState state,
+ Configuration conf) {
+ if (!state.getApplicationState().isEmpty()) {
+ long waitTime =
+ conf.getLong(YarnConfiguration.RM_SCHEDULER_RECOVERY_WAIT_MS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_RECOVERY_WAIT_MS);
+ rmContext.setSchedulerRecoveryStartAndWaitTime(waitTime);
+ }
+ }
+
/**
* Retrieve RM bind address from configuration
*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 5de407d..23ea661 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -799,6 +799,11 @@ private synchronized void nodeUpdate(RMNode nm) {
}
private synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
+ if (rmContext.isWorkPreservingRecoveryEnabled()
+ && !rmContext.isSchedulerReadyForAllocatingContainers()) {
+ LOG.info("Skip allocating containers. Scheduler is waiting for recovery.");
+ return;
+ }
// Assign new containers...
// 1. Check for reserved applications
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 5725f8c..bc5ab93 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1003,6 +1003,12 @@ public int compare(NodeId n1, NodeId n2) {
}
private synchronized void attemptScheduling(FSSchedulerNode node) {
+ if (rmContext.isWorkPreservingRecoveryEnabled()
+ && !rmContext.isSchedulerReadyForAllocatingContainers()) {
+ LOG.info("Skip allocating containers. Scheduler is waiting for recovery.");
+ return;
+ }
+
// Assign new containers...
// 1. Check for reserved applications
// 2. Schedule if there are no reservations
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 4681516..3f1fc26 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -708,6 +708,13 @@ private synchronized void nodeUpdate(RMNode rmNode) {
completedContainer, RMContainerEventType.FINISHED);
}
+
+ if (rmContext.isWorkPreservingRecoveryEnabled()
+ && !rmContext.isSchedulerReadyForAllocatingContainers()) {
+ LOG.info("Skip allocating containers. Scheduler is waiting for recovery.");
+ return;
+ }
+
if (Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
node.getAvailableResource(),minimumAllocation)) {
LOG.debug("Node heartbeat " + rmNode.getNodeID() +