diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index fd11aea..822debf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -369,6 +369,17 @@ private static ApplicationReportExt convertToApplicationReport( queue = eventInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO) .toString(); } else if (event.getEventType().equals( + ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) { + Map eventInfo = event.getEventInfo(); + if (eventInfo == null) { + continue; + } + if (eventInfo.containsKey( + ApplicationMetricsConstants.STATE_EVENT_INFO)) { + state = YarnApplicationState.valueOf(eventInfo.get( + ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); + } + } else if (event.getEventType().equals( ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) { progress=1.0F; finishedTime = event.getTimestamp(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 75dce07..deae894 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -274,12 +274,8 @@ public Response getLogs(@Context HttpServletRequest req, } } - // TODO: YARN-5029. RM would send the update event. We could get - // the consistent YarnApplicationState. - // Will remove YarnApplicationState.ACCEPTED. private boolean isRunningState(YarnApplicationState appState) { - return appState == YarnApplicationState.ACCEPTED - || appState == YarnApplicationState.RUNNING; + return appState == YarnApplicationState.RUNNING; } private boolean isFinishedState(YarnApplicationState appState) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index c456e54..d06b7cb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -40,6 +40,9 @@ public static final String UPDATED_EVENT_TYPE = "YARN_APPLICATION_UPDATED"; + public static final String STATE_UPDATED_EVENT_TYPE = + "YARN_APPLICATION_STATE_UPDATED"; + public static final String NAME_ENTITY_INFO = "YARN_APPLICATION_NAME"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicaitonStateUpdatedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicaitonStateUpdatedEvent.java new file mode 100644 index 0000000..73d7253 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicaitonStateUpdatedEvent.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.metrics; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; + +public class ApplicaitonStateUpdatedEvent extends SystemMetricsEvent{ + private ApplicationId appId; + private YarnApplicationState appState; + + public ApplicaitonStateUpdatedEvent(ApplicationId appliocationId, + YarnApplicationState state, long updatedTime) { + super(SystemMetricsEventType.APP_STATE_UPDATED, updatedTime); + this.appId = appliocationId; + this.appState = state; + } + + public ApplicationId getApplicationId() { + return appId; + } + + public YarnApplicationState getAppState() { + return appState; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsEventType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsEventType.java index c11034e..fcda4b4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsEventType.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsEventType.java @@ -25,6 +25,7 @@ APP_FINISHED, APP_ACLS_UPDATED, APP_UPDATED, + APP_STATE_UPDATED, // app attempt events APP_ATTEMPT_REGISTERED, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 84a3b19..52aba85 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; @@ -158,6 +159,18 @@ public void appACLsUpdated(RMApp app, String appViewACLs, } @SuppressWarnings("unchecked") + public void appStateUpdated(RMApp app, YarnApplicationState appState, + long updatedTime) { + if (publishSystemMetrics) { + dispatcher.getEventHandler().handle( + new ApplicaitonStateUpdatedEvent( + app.getApplicationId(), + appState, + updatedTime)); + } + } + + @SuppressWarnings("unchecked") public void appAttemptRegistered(RMAppAttempt appAttempt, long registeredTime) { if (publishSystemMetrics) { @@ -247,6 +260,10 @@ protected void handleSystemMetricsEvent( case APP_UPDATED: publishApplicationUpdatedEvent((ApplicationUpdatedEvent) event); break; + case APP_STATE_UPDATED: + publishApplicationStateUpdatedEvent( + (ApplicaitonStateUpdatedEvent)event); + break; case APP_ATTEMPT_REGISTERED: publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event); break; @@ -352,6 +369,20 @@ private void publishApplicationUpdatedEvent(ApplicationUpdatedEvent event) { putEntity(entity); } + private void publishApplicationStateUpdatedEvent( + ApplicaitonStateUpdatedEvent event) { + TimelineEntity entity = createApplicationEntity(event.getApplicationId()); + Map eventInfo = new HashMap(); + eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, + event.getAppState()); + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setEventType(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); + tEvent.setTimestamp(event.getTimestamp()); + tEvent.setEventInfo(eventInfo); + entity.addEvent(tEvent); + putEntity(entity); + } + private void publishApplicationACLsUpdatedEvent( ApplicationACLsUpdatedEvent event) { TimelineEntity entity = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 10c9edc..d7ea963 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -250,7 +250,8 @@ .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING, - RMAppEventType.ATTEMPT_REGISTERED) + RMAppEventType.ATTEMPT_REGISTERED, new RMAppStateUpdateTransition( + YarnApplicationState.RUNNING)) .addTransition(RMAppState.ACCEPTED, EnumSet.of(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING), // ACCEPTED state is possible to receive ATTEMPT_FAILED/ATTEMPT_FINISHED @@ -928,7 +929,21 @@ public void transition(RMAppImpl app, RMAppEvent event) { nodeUpdateEvent.getNode()); }; } - + + private static final class RMAppStateUpdateTransition + extends RMAppTransition { + private YarnApplicationState stateToATS; + + public RMAppStateUpdateTransition(YarnApplicationState state) { + stateToATS = state; + } + + public void transition(RMAppImpl app, RMAppEvent event) { + app.rmContext.getSystemMetricsPublisher().appStateUpdated( + app, stateToATS, app.systemClock.getTime()); + }; + } + private static final class AppRunningOnNodeTransition extends RMAppTransition { public void transition(RMAppImpl app, RMAppEvent event) { RMAppRunningOnNodeEvent nodeAddedEvent = (RMAppRunningOnNodeEvent) event; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index f2b02bc..ddd10dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -106,6 +106,7 @@ public static void tearDown() throws Exception { @Test(timeout = 10000) public void testPublishApplicationMetrics() throws Exception { + long stateUpdateTimeStamp = System.currentTimeMillis(); for (int i = 1; i <= 2; ++i) { ApplicationId appId = ApplicationId.newInstance(0, i); RMApp app = createRMApp(appId); @@ -121,6 +122,8 @@ public void testPublishApplicationMetrics() throws Exception { } else { metricsPublisher.appUpdated(app, 4L); } + metricsPublisher.appStateUpdated(app, YarnApplicationState.RUNNING, + stateUpdateTimeStamp); metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime()); if (i == 1) { metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L); @@ -128,6 +131,9 @@ public void testPublishApplicationMetrics() throws Exception { // in case user doesn't specify the ACLs metricsPublisher.appACLsUpdated(app, null, 4L); } + // Sleep 1s to make sure all the putEntity calls have been handled + // by timelineStore + Thread.sleep(1000); TimelineEntity entity = null; do { entity = @@ -212,6 +218,7 @@ public void testPublishApplicationMetrics() throws Exception { boolean hasUpdatedEvent = false; boolean hasFinishedEvent = false; boolean hasACLsUpdatedEvent = false; + boolean hasStateUpdateEvent = false; for (TimelineEvent event : entity.getEvents()) { if (event.getEventType().equals( ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { @@ -249,10 +256,21 @@ public void testPublishApplicationMetrics() throws Exception { ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) { hasACLsUpdatedEvent = true; Assert.assertEquals(4L, event.getTimestamp()); + } else if (event.getEventType().equals( + ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) { + hasStateUpdateEvent = true; + Assert.assertEquals(event.getTimestamp(), stateUpdateTimeStamp); + Assert.assertEquals(YarnApplicationState.RUNNING.toString(), event + .getEventInfo().get( + ApplicationMetricsConstants.STATE_EVENT_INFO)); } } - Assert.assertTrue(hasCreatedEvent && hasFinishedEvent - && hasACLsUpdatedEvent && hasUpdatedEvent); + // Do assertTrue verification separately for easier debug + Assert.assertTrue(hasCreatedEvent); + Assert.assertTrue(hasFinishedEvent); + Assert.assertTrue(hasACLsUpdatedEvent); + Assert.assertTrue(hasUpdatedEvent); + Assert.assertTrue(hasStateUpdateEvent); } }