From bea4c0e660714ea76fbc02a2bfd3ead33d723693 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 18 Feb 2019 23:46:57 +0530 Subject: [PATCH] YARN-8132 --- .../rmapp/attempt/RMAppAttemptImpl.java | 10 +++-- .../rmapp/TestRMAppTransitions.java | 44 +++++++++++++--------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 03039da..1691814 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1372,7 +1372,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, setTrackingUrlToRMAppPage(stateToBeStored); } String finalTrackingUrl = getOriginalTrackingUrl(); - FinalApplicationStatus finalStatus = null; + FinalApplicationStatus status = null; int exitStatus = ContainerExitStatus.INVALID; switch (event.getType()) { case LAUNCH_FAILED: @@ -1387,7 +1387,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, diags.append(unregisterEvent.getDiagnosticMsg()); // reset finalTrackingUrl to url sent by am finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - finalStatus = unregisterEvent.getFinalApplicationStatus(); + status = unregisterEvent.getFinalApplicationStatus(); break; case CONTAINER_FINISHED: RMAppAttemptContainerFinishedEvent finishEvent = @@ -1396,8 +1396,10 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, exitStatus = finishEvent.getContainerStatus().getExitStatus(); break; case KILL: + status = FinalApplicationStatus.KILLED; break; case FAIL: + status = FinalApplicationStatus.FAILED; diags.append(event.getDiagnosticMsg()); break; case EXPIRE: @@ -1414,13 +1416,14 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, + stateToBeStored, finalTrackingUrl, diags.toString(), status, exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(), this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); rmStore.updateApplicationAttemptState(attemptState); + finalStatus = status; } private static class FinalSavingTransition extends BaseTransition { @@ -1916,7 +1919,6 @@ private void updateInfoOnAMUnregister(RMAppAttemptEvent event) { (RMAppAttemptUnregistrationEvent) event; this.diagnostics.append(unregisterEvent.getDiagnosticMsg()); originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - finalStatus = unregisterEvent.getFinalApplicationStatus(); } private static final class ContainerFinishedTransition diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 32dcd3b..eab422f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; @@ -722,7 +723,7 @@ public void testAppNewKill() throws IOException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateNotSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -741,7 +742,7 @@ public void testAppNewReject() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -759,7 +760,7 @@ public void testAppNewRejectAddToStore() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); rmContext.getStateStore().removeApplication(application); } @@ -781,7 +782,7 @@ public void testAppNewSavingKill() throws IOException { rmDispatcher.await(); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -800,7 +801,7 @@ public void testAppNewSavingReject() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -817,7 +818,7 @@ public void testAppNewSavingSaveReject() throws IOException { assertFailed(application, rejectedText); verify(store, times(0)).updateApplicationState( any(ApplicationStateData.class)); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); assertTimesAtFinish(application); } @@ -835,7 +836,7 @@ public void testAppSubmittedRejected() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -857,7 +858,7 @@ public void testAppSubmittedKill() throws IOException, InterruptedException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -894,7 +895,7 @@ public void testAppAcceptedFailed() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, ".*" + message + ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); } @Test @@ -921,7 +922,7 @@ public void testAppAcceptedKill() throws IOException, InterruptedException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -945,7 +946,7 @@ public void testAppAcceptedAttemptKilled() throws IOException, sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); } @@ -969,7 +970,7 @@ public void testAppRunningKill() throws IOException { sendAttemptUpdateSavedEvent(application); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1028,7 +1029,7 @@ public void testAppRunningFailed() throws IOException { rmDispatcher.await(); assertFailed(application, ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1090,7 +1091,9 @@ public void testAppFinishedFinished() throws Exception { StringBuilder diag = application.getDiagnostics(); Assert.assertEquals("application diagnostics is not correct", "", diag.toString()); - verifyApplicationFinished(RMAppState.FINISHED); + // finished without a proper final state is the same as failed + verifyApplicationFinished(RMAppState.FINISHED, + FinalApplicationStatus.FAILED); verifyAppRemovedSchedulerEvent(application, RMAppState.FINISHED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1118,7 +1121,7 @@ public void testAppFailedFailed() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); @@ -1173,7 +1176,7 @@ public void testAppKilledKilled() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -1309,15 +1312,20 @@ private void verifyAppCompletedEvent(RMApp app) { rmAppManagerEvent.getApplicationId().getId()); } - private void verifyApplicationFinished(RMAppState state) { + private void verifyApplicationFinished(RMAppState state, + FinalApplicationStatus finalAppStatus) { ArgumentCaptor finalState = ArgumentCaptor.forClass(RMAppState.class); verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); finalState = ArgumentCaptor.forClass(RMAppState.class); - verify(publisher).appFinished(any(RMApp.class), finalState.capture(), + ArgumentCaptor app = + ArgumentCaptor.forClass(RMApp.class); + verify(publisher).appFinished(app.capture(), finalState.capture(), anyLong()); Assert.assertEquals(state, finalState.getValue()); + Assert.assertEquals(finalAppStatus, + app.getValue().getFinalApplicationStatus()); } private void verifyAppRemovedSchedulerEvent(RMApp app, -- 2.7.4 (Apple Git-66)