diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java index eadb5b7..8eacc73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java @@ -41,6 +41,12 @@ public static final String FINISHED_IN_RM_EVENT_TYPE = "YARN_RM_CONTAINER_FINISHED"; + public static final String CONTAINER_CREATED_TIME = + "YARN_CONTAINER_CREATED_TIME"; + + public static final String CONTAINER_FINISHED_TIME = + "YARN_CONTAINER_FINISHED_TIME"; + public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT"; public static final String ALLOCATED_MEMORY_ENTITY_INFO = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index 387900a..92737d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -30,8 +29,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier; @@ -71,10 +68,6 @@ private Context context; - private NodeId nodeId; - - private String httpAddress; - private final Map appToClientMap; public NMTimelinePublisher(Context context) { @@ -92,15 +85,6 @@ protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); } - @Override - protected void serviceStart() throws Exception { - super.serviceStart(); - // context will be updated after containerManagerImpl is started - // hence NMMetricsPublisher is added subservice of containerManagerImpl - this.nodeId = context.getNodeId(); - this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort(); - } - @VisibleForTesting Map getAppToClientMap() { return appToClientMap; @@ -163,24 +147,6 @@ public void reportContainerResourceUsage(Container container, Long pmemUsage, private void publishContainerCreatedEvent(ContainerEvent event) { ContainerId containerId = event.getContainerID(); ContainerEntity entity = createContainerEntity(containerId); - Container container = context.getContainers().get(containerId); - Resource resource = container.getResource(); - - Map entityInfo = new HashMap(); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, - resource.getMemorySize()); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, - resource.getVirtualCores()); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, - nodeId.getHost()); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, - nodeId.getPort()); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, - container.getPriority().toString()); - entityInfo.put( - ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, - httpAddress); - entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE); @@ -198,16 +164,9 @@ private void publishContainerFinishedEvent(ContainerStatus containerStatus, ContainerId containerId = containerStatus.getContainerId(); TimelineEntity entity = createContainerEntity(containerId); - Map eventInfo = new HashMap(); - eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, - containerStatus.getDiagnostics()); - eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, - containerStatus.getExitStatus()); - TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(timeStamp); - tEvent.setInfo(eventInfo); entity.addEvent(tEvent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index a248199..b50554c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -152,22 +152,22 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(finishedTime); - Map eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, app.getDiagnostics().toString()); - eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, app.getFinalApplicationStatus().toString()); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, RMServerUtils.createApplicationState(state).toString()); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() == null ? null : app.getCurrentAppAttempt().getAppAttemptId(); if (appAttemptId != null) { - eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, + entityInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, appAttemptId.toString()); } - tEvent.setInfo(eventInfo); - - entity.addEvent(tEvent); + entity.setInfo(entityInfo); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); @@ -179,15 +179,16 @@ public void appStateUpdated(RMApp app, YarnApplicationState appState, long updatedTime) { ApplicationEntity entity = createApplicationEntity(app.getApplicationId()); - Map eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, - appState); + TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); tEvent.setTimestamp(updatedTime); - tEvent.setInfo(eventInfo); entity.addEvent(tEvent); + Map entityInfo = new HashMap(); + entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, appState); + tEvent.setInfo(entityInfo); + getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); } @@ -245,21 +246,23 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); tEvent.setTimestamp(registeredTime); - Map eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, appAttempt.getTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, appAttempt.getOriginalTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, appAttempt.getHost()); - eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, appAttempt.getRpcPort()); if (appAttempt.getMasterContainer() != null) { - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, appAttempt.getMasterContainer().getId().toString()); } - tEvent.setInfo(eventInfo); - entity.addEvent(tEvent); + entity.setInfo(entityInfo); + getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, entity, appAttempt.getAppAttemptId().getApplicationId())); @@ -276,26 +279,28 @@ public void appAttemptFinished(RMAppAttempt appAttempt, TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(finishedTime); - Map eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, appAttempt.getTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, appAttempt.getOriginalTrackingUrl()); - eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, appAttempt.getDiagnostics()); // app will get the final status from app attempt, or create one // based on app state if it doesn't exist - eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, app.getFinalApplicationStatus().toString()); - eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils + entityInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); if (appAttempt.getMasterContainer() != null) { - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + entityInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, appAttempt.getMasterContainer().getId().toString()); } - tEvent.setInfo(eventInfo); + entity.setInfo(entityInfo); + - entity.addEvent(tEvent); getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, entity, appAttempt.getAppAttemptId().getApplicationId())); @@ -320,25 +325,27 @@ public void containerCreated(RMContainer container, long createdTime) { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE); tEvent.setTimestamp(createdTime); - // updated as event info instead of entity info, as entity info is updated - // by NM - Map eventInfo = new HashMap(); - eventInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, container.getAllocatedResource().getMemorySize()); - eventInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, + entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, container.getAllocatedResource().getVirtualCores()); - eventInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, container.getAllocatedNode().getHost()); - eventInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, container.getAllocatedNode().getPort()); - eventInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, container.getAllocatedPriority().getPriority()); - eventInfo.put( + entityInfo.put( ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, container.getNodeHttpAddress()); - tEvent.setInfo(eventInfo); + entityInfo.put(ContainerMetricsConstants.CONTAINER_CREATED_TIME, + createdTime); + entity.setInfo(entityInfo); + - entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, container .getContainerId().getApplicationAttemptId().getApplicationId())); @@ -354,22 +361,19 @@ public void containerFinished(RMContainer container, long finishedTime) { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.FINISHED_IN_RM_EVENT_TYPE); tEvent.setTimestamp(finishedTime); - Map eventInfo = new HashMap(); - eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, + entity.addEvent(tEvent); + + Map entityInfo = new HashMap(); + entityInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, container.getDiagnosticsInfo()); - eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, + entityInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, container.getContainerExitStatus()); - eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, + entityInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, container.getContainerState().toString()); - Map entityInfo = new HashMap(); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, - container.getAllocatedNode().getHost()); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, - container.getAllocatedNode().getPort()); + entityInfo.put(ContainerMetricsConstants.CONTAINER_FINISHED_TIME, + finishedTime); entity.setInfo(entityInfo); - tEvent.setInfo(eventInfo); - entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, container .getContainerId().getApplicationAttemptId().getApplicationId()));