diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a18ef7c..8cc0703 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -402,7 +402,7 @@ private static void addDeprecatedKeys() {
public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
- public static final boolean DEFAULT_YARN_FAIL_FAST = true;
+ public static final boolean DEFAULT_YARN_FAIL_FAST = false;
public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 62ba599..7070cd7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -343,9 +343,12 @@
Should YARN fail fast if it encounters any errors.
+ This is a global config for all other components including RM,NM etc.
+ If no value is set for component-specific config (e.g yarn.resourcemanager.fail-fast),
+ this value will be the default.
yarn.fail-fast
- true
+ false
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index b7f1e6c..6e3aa69 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -45,6 +45,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -190,11 +191,12 @@ public void transition(RMStateStore store, RMStateStoreEvent event) {
LOG.info("Storing info for app: " + appId);
try {
store.storeApplicationStateInternal(appId, appState);
- store.notifyApplication(new RMAppEvent(appId,
- RMAppEventType.APP_NEW_SAVED));
} catch (Exception e) {
LOG.error("Error storing app: " + appId, e);
store.notifyStoreOperationFailed(e);
+ } finally {
+ store.notifyApplication(new RMAppEvent(appId,
+ RMAppEventType.APP_NEW_SAVED));
}
};
}
@@ -215,13 +217,14 @@ public void transition(RMStateStore store, RMStateStoreEvent event) {
LOG.info("Updating info for app: " + appId);
try {
store.updateApplicationStateInternal(appId, appState);
+ } catch (Exception e) {
+ LOG.error("Error updating app: " + appId, e);
+ store.notifyStoreOperationFailed(e);
+ } finally {
if (((RMStateUpdateAppEvent) event).isNotifyApplication()) {
store.notifyApplication(new RMAppEvent(appId,
RMAppEventType.APP_UPDATE_SAVED));
}
- } catch (Exception e) {
- LOG.error("Error updating app: " + appId, e);
- store.notifyStoreOperationFailed(e);
}
};
}
@@ -266,12 +269,13 @@ public void transition(RMStateStore store, RMStateStoreEvent event) {
}
store.storeApplicationAttemptStateInternal(attemptState.getAttemptId(),
attemptState);
- store.notifyApplicationAttempt(new RMAppAttemptEvent
- (attemptState.getAttemptId(),
- RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
} catch (Exception e) {
LOG.error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
store.notifyStoreOperationFailed(e);
+ } finally {
+ store.notifyApplicationAttempt(new RMAppAttemptEvent
+ (attemptState.getAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
}
};
}
@@ -293,12 +297,13 @@ public void transition(RMStateStore store, RMStateStoreEvent event) {
}
store.updateApplicationAttemptStateInternal(attemptState.getAttemptId(),
attemptState);
- store.notifyApplicationAttempt(new RMAppAttemptEvent
- (attemptState.getAttemptId(),
- RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
} catch (Exception e) {
LOG.error("Error updating appAttempt: " + attemptState.getAttemptId(), e);
store.notifyStoreOperationFailed(e);
+ } finally {
+ store.notifyApplicationAttempt(new RMAppAttemptEvent
+ (attemptState.getAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
}
};
}
@@ -1013,18 +1018,20 @@ protected void handleStoreEvent(RMStateStoreEvent event) {
*/
protected void notifyStoreOperationFailed(Exception failureCause) {
LOG.error("State store operation failed ", failureCause);
- if (failureCause instanceof StoreFencedException) {
+ if (HAUtil.isHAEnabled(getConfig())) {
+ LOG.warn("State-store fenced ! Transitioning RM to standby");
updateFencedState();
Thread standByTransitionThread =
new Thread(new StandByTransitionThread());
standByTransitionThread.setName("StandByTransitionThread Handler");
standByTransitionThread.start();
+ } else if (YarnConfiguration.shouldRMFailFast(getConfig())) {
+ LOG.warn("Fail RM now due to state-store error!");
+ rmDispatcher.getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
+ failureCause));
} else {
- if (YarnConfiguration.shouldRMFailFast(getConfig())) {
- rmDispatcher.getEventHandler().handle(
- new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
- failureCause));
- }
+ LOG.warn("Skip the state-store error.");
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 0550087..945405e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -1049,7 +1049,7 @@ public void run() {
LOG.info(VerifyActiveStatusThread.class.getName() + " thread " +
"interrupted! Exiting!");
} catch (Exception e) {
- notifyStoreOperationFailed(new StoreFencedException());
+ notifyStoreOperationFailed(e);
}
}
}