diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java index eadb5b7..8eacc73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java @@ -41,6 +41,12 @@ public static final String FINISHED_IN_RM_EVENT_TYPE = "YARN_RM_CONTAINER_FINISHED"; + public static final String CONTAINER_CREATED_TIME = + "YARN_CONTAINER_CREATED_TIME"; + + public static final String CONTAINER_FINISHED_TIME = + "YARN_CONTAINER_FINISHED_TIME"; + public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT"; public static final String ALLOCATED_MEMORY_ENTITY_INFO = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index 387900a..846f543 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -98,7 +98,6 @@ protected void serviceStart() throws Exception { // context will be updated after containerManagerImpl is started // hence NMMetricsPublisher is added subservice of containerManagerImpl this.nodeId = context.getNodeId(); - this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort(); } @VisibleForTesting @@ -180,6 +179,8 @@ private void publishContainerCreatedEvent(ContainerEvent event) { entityInfo.put( ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, httpAddress); + entityInfo.put(ContainerMetricsConstants.CONTAINER_CREATED_TIME, + event.getTimestamp()); entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); @@ -198,17 +199,18 @@ private void publishContainerFinishedEvent(ContainerStatus containerStatus, ContainerId containerId = containerStatus.getContainerId(); TimelineEntity entity = createContainerEntity(containerId); - Map eventInfo = new HashMap(); - eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + Map entityInfo = new HashMap(); + entityInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, containerStatus.getDiagnostics()); - eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, + entityInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, containerStatus.getExitStatus()); + entityInfo.put(ContainerMetricsConstants.CONTAINER_FINISHED_TIME, + timeStamp); + entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(timeStamp); - tEvent.setInfo(eventInfo); - entity.addEvent(tEvent); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, @@ -302,6 +304,11 @@ public void publishApplicationEvent(ApplicationEvent event) { public void publishContainerEvent(ContainerEvent event) { // publish only when the desired event is received + if (this.httpAddress != null) { + // update httpAddress for first time. When this service started, + // web server will not be started. + this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort(); + } switch (event.getType()) { case INIT_CONTAINER: publishContainerCreatedEvent(event); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index a248199..09725ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -152,22 +152,22 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(finishedTime); - Map eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, app.getDiagnostics().toString()); - eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, app.getFinalApplicationStatus().toString()); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, RMServerUtils.createApplicationState(state).toString()); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() == null ? null : app.getCurrentAppAttempt().getAppAttemptId(); if (appAttemptId != null) { - eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, appAttemptId.toString()); } - tEvent.setInfo(eventInfo); - - entity.addEvent(tEvent); + entity.setInfo(entityInfo); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); @@ -179,15 +179,16 @@ public void appStateUpdated(RMApp app, YarnApplicationState appState, long updatedTime) { ApplicationEntity entity = createApplicationEntity(app.getApplicationId()); - Map eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, - appState); + TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); tEvent.setTimestamp(updatedTime); - tEvent.setInfo(eventInfo); entity.addEvent(tEvent); + Map entityInfo = new HashMap(); + entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, appState); + tEvent.setInfo(entityInfo); + getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); } @@ -245,21 +246,23 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); tEvent.setTimestamp(registeredTime); - Map eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, appAttempt.getTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, appAttempt.getOriginalTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, appAttempt.getHost()); - eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, appAttempt.getRpcPort()); if (appAttempt.getMasterContainer() != null) { - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, appAttempt.getMasterContainer().getId().toString()); } - tEvent.setInfo(eventInfo); - entity.addEvent(tEvent); + entity.setInfo(entityInfo); + getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, entity, appAttempt.getAppAttemptId().getApplicationId())); @@ -276,26 +279,28 @@ public void appAttemptFinished(RMAppAttempt appAttempt, TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(finishedTime); - Map eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, appAttempt.getTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, appAttempt.getOriginalTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, appAttempt.getDiagnostics()); // app will get the final status from app attempt, or create one // based on app state if it doesn't exist - eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, app.getFinalApplicationStatus().toString()); - eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils + entityInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); if (appAttempt.getMasterContainer() != null) { - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, appAttempt.getMasterContainer().getId().toString()); } - tEvent.setInfo(eventInfo); + entity.setInfo(entityInfo); + - entity.addEvent(tEvent); getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, entity, appAttempt.getAppAttemptId().getApplicationId()));