diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 29c5953..9785223 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; @@ -197,6 +198,28 @@ public static YarnApplicationState createApplicationState( } } + public static FinalApplicationStatus createFinalApplicationStatus( + RMAppState rmAppState) { + switch(rmAppState) { + case NEW: + case NEW_SAVING: + case SUBMITTED: + case ACCEPTED: + case RUNNING: + case FINAL_SAVING: + case KILLING: + return FinalApplicationStatus.UNDEFINED; + // finished without a proper final state is the same as failed + case FINISHING: + case FINISHED: + case FAILED: + return FinalApplicationStatus.FAILED; + case KILLED: + return FinalApplicationStatus.KILLED; + } + throw new YarnRuntimeException("Unknown state passed!"); + } + public static YarnApplicationAttemptState createApplicationAttemptState( RMAppAttemptState rmAppAttemptState) { switch (rmAppAttemptState) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 886f57d..3e85588 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -143,7 +143,8 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, @SuppressWarnings("unchecked") public void appAttemptFinished(RMAppAttempt appAttempt, - RMAppAttemptState state, long finishedTime) { + RMAppAttemptState appAttemtpState, RMAppState appState, + long finishedTime) { if (publishSystemMetrics) { dispatcher.getEventHandler().handle( new AppAttemptFinishedEvent( @@ -151,8 +152,10 @@ public void appAttemptFinished(RMAppAttempt appAttempt, appAttempt.getTrackingUrl(), appAttempt.getOriginalTrackingUrl(), appAttempt.getDiagnostics(), - appAttempt.getFinalApplicationStatus(), - RMServerUtils.createApplicationAttemptState(state), + (appAttempt.getFinalApplicationStatus() == null ? + RMServerUtils.createFinalApplicationStatus(appState) : + appAttempt.getFinalApplicationStatus()), + RMServerUtils.createApplicationAttemptState(appAttemtpState), finishedTime)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 815b86a..88bf83c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -405,8 +405,8 @@ public FinalApplicationStatus getFinalApplicationStatus() { && currentAttempt.getFinalApplicationStatus() != null) { return currentAttempt.getFinalApplicationStatus(); } - return - createFinalApplicationStatus(this.stateMachine.getCurrentState()); + return RMServerUtils.createFinalApplicationStatus( + this.stateMachine.getCurrentState()); } finally { this.readLock.unlock(); } @@ -489,27 +489,6 @@ public RMAppAttempt getCurrentAppAttempt() { } } - private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) { - switch(state) { - case NEW: - case NEW_SAVING: - case SUBMITTED: - case ACCEPTED: - case RUNNING: - case FINAL_SAVING: - case KILLING: - return FinalApplicationStatus.UNDEFINED; - // finished without a proper final state is the same as failed - case FINISHING: - case FINISHED: - case FAILED: - return FinalApplicationStatus.FAILED; - case KILLED: - return FinalApplicationStatus.KILLED; - } - throw new YarnRuntimeException("Unknown state passed!"); - } - @Override public int pullRMNodeUpdates(Collection updatedNodes) { this.writeLock.lock(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 863130f..18e0dc4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1159,8 +1159,10 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.rmContext.getRMApplicationHistoryWriter() .applicationAttemptFinished(appAttempt, finalAttemptState); appAttempt.rmContext.getSystemMetricsPublisher() - .appAttemptFinished( - appAttempt, finalAttemptState, System.currentTimeMillis()); + .appAttemptFinished(appAttempt, finalAttemptState, + appAttempt.rmContext.getRMApps().get( + appAttempt.applicationAttemptId.getApplicationId()).getState(), + System.currentTimeMillis()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index ec20af3..24033d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -161,68 +161,76 @@ public void testPublishApplicationMetrics() throws Exception { @Test(timeout = 10000) public void testPublishAppAttemptMetrics() throws Exception { - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); - RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId); - metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L); - metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, - Integer.MAX_VALUE + 2L); - TimelineEntity entity = null; - do { - entity = - store.getEntity(appAttemptId.toString(), - AppAttemptMetricsConstants.ENTITY_TYPE, - EnumSet.allOf(Field.class)); - // ensure two events are both published before leaving the loop - } while (entity == null || entity.getEvents().size() < 2); - // verify all the fields - Assert.assertEquals(AppAttemptMetricsConstants.ENTITY_TYPE, - entity.getEntityType()); - Assert.assertEquals(appAttemptId.toString(), entity.getEntityId()); - Assert.assertEquals( - appAttemptId.getApplicationId().toString(), - entity.getPrimaryFilters() - .get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator() - .next()); - boolean hasRegisteredEvent = false; - boolean hasFinishedEvent = false; - for (TimelineEvent event : entity.getEvents()) { - if (event.getEventType().equals( - AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) { - hasRegisteredEvent = true; - Assert.assertEquals(appAttempt.getHost(), - event.getEventInfo() - .get(AppAttemptMetricsConstants.HOST_EVENT_INFO)); - Assert - .assertEquals(appAttempt.getRpcPort(), + for (int i = 0; i < 2; ++i) { + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1 + i); + RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, i == 1); + metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L); + metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, + RMAppState.FINISHED, Integer.MAX_VALUE + 2L); + TimelineEntity entity = null; + do { + entity = + store.getEntity(appAttemptId.toString(), + AppAttemptMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + // ensure two events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 2); + // verify all the fields + Assert.assertEquals(AppAttemptMetricsConstants.ENTITY_TYPE, + entity.getEntityType()); + Assert.assertEquals(appAttemptId.toString(), entity.getEntityId()); + Assert.assertEquals( + appAttemptId.getApplicationId().toString(), + entity.getPrimaryFilters() + .get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator() + .next()); + boolean hasRegisteredEvent = false; + boolean hasFinishedEvent = false; + for (TimelineEvent event : entity.getEvents()) { + if (event.getEventType().equals( + AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) { + hasRegisteredEvent = true; + Assert.assertEquals(appAttempt.getHost(), + event.getEventInfo() + .get(AppAttemptMetricsConstants.HOST_EVENT_INFO)); + Assert + .assertEquals(appAttempt.getRpcPort(), + event.getEventInfo().get( + AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO)); + Assert.assertEquals( + appAttempt.getMasterContainer().getId().toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)); + } else if (event.getEventType().equals( + AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) { + hasFinishedEvent = true; + Assert.assertEquals(appAttempt.getDiagnostics(), event.getEventInfo() + .get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)); + Assert.assertEquals(appAttempt.getTrackingUrl(), event.getEventInfo() + .get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO)); + Assert.assertEquals( + appAttempt.getOriginalTrackingUrl(), + event.getEventInfo().get( + AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO)); + if (i == 0) { + Assert.assertEquals( + appAttempt.getFinalApplicationStatus().toString(), event.getEventInfo().get( - AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO)); - Assert.assertEquals( - appAttempt.getMasterContainer().getId().toString(), - event.getEventInfo().get( - AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)); - } else if (event.getEventType().equals( - AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) { - hasFinishedEvent = true; - Assert.assertEquals(appAttempt.getDiagnostics(), event.getEventInfo() - .get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)); - Assert.assertEquals(appAttempt.getTrackingUrl(), event.getEventInfo() - .get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO)); - Assert.assertEquals( - appAttempt.getOriginalTrackingUrl(), - event.getEventInfo().get( - AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO)); - Assert.assertEquals( - appAttempt.getFinalApplicationStatus().toString(), - event.getEventInfo().get( - AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO)); - Assert.assertEquals( - YarnApplicationAttemptState.FINISHED.toString(), - event.getEventInfo().get( - AppAttemptMetricsConstants.STATE_EVENT_INFO)); + AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO)); + } else { + Assert.assertEquals(FinalApplicationStatus.FAILED.toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO)); + } + Assert.assertEquals( + YarnApplicationAttemptState.FINISHED.toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.STATE_EVENT_INFO)); + } } + Assert.assertTrue(hasRegisteredEvent && hasFinishedEvent); } - Assert.assertTrue(hasRegisteredEvent && hasFinishedEvent); } @Test(timeout = 10000) @@ -318,7 +326,7 @@ private static RMApp createRMApp(ApplicationId appId) { } private static RMAppAttempt createRMAppAttempt( - ApplicationAttemptId appAttemptId) { + ApplicationAttemptId appAttemptId, boolean missingFinalStatus) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId); when(appAttempt.getHost()).thenReturn("test host"); @@ -331,8 +339,12 @@ private static RMAppAttempt createRMAppAttempt( when(appAttempt.getTrackingUrl()).thenReturn("test tracking url"); when(appAttempt.getOriginalTrackingUrl()).thenReturn( "test original tracking url"); - when(appAttempt.getFinalApplicationStatus()).thenReturn( - FinalApplicationStatus.UNDEFINED); + if (missingFinalStatus) { + when(appAttempt.getFinalApplicationStatus()).thenReturn(null); + } else { + when(appAttempt.getFinalApplicationStatus()).thenReturn( + FinalApplicationStatus.UNDEFINED); + } return appAttempt; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 6608ccd..e14f90a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; @@ -1385,7 +1386,7 @@ private void verifyApplicationAttemptFinished(RMAppAttemptState state) { finalState = ArgumentCaptor.forClass(RMAppAttemptState.class); verify(publisher).appAttemptFinished(any(RMAppAttempt.class), finalState.capture(), - anyLong()); + any(RMAppState.class), anyLong()); Assert.assertEquals(state, finalState.getValue()); }