From 816ac1618272ebd7ee9de0d7371062618fcdf4b8 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 29 Mar 2019 13:28:14 +0530 Subject: [PATCH] YARN-9418 --- .../timelineservice/NMTimelinePublisher.java | 8 ++------ .../timelineservice/TestNMTimelinePublisher.java | 18 ++++++++++++------ .../metrics/TimelineServiceV2Publisher.java | 2 ++ .../metrics/TestSystemMetricsPublisherForV2.java | 3 ++- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index 79443f4..b2d9376 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -186,8 +186,6 @@ public void reportContainerResourceUsage(Container container, Long pmemUsage, Math.round(cpuUsagePercentPerCore)); entity.addMetric(cpuMetric); } - entity.setIdPrefix(TimelineServiceHelper. - invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId(). getApplicationAttemptId().getApplicationId(); try { @@ -249,7 +247,6 @@ private void publishContainerCreatedEvent(ContainerEvent event) { long containerStartTime = container.getContainerStartTime(); entity.addEvent(tEvent); entity.setCreatedTime(containerStartTime); - entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); } @@ -277,7 +274,6 @@ private void publishContainerFinishedEvent(ContainerStatus containerStatus, tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(containerFinishTime); entity.addEvent(tEvent); - entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); @@ -295,8 +291,6 @@ private void publishContainerLocalizationEvent( tEvent.setId(eventType); tEvent.setTimestamp(event.getTimestamp()); entity.addEvent(tEvent); - entity.setIdPrefix(TimelineServiceHelper. - invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId(). getApplicationAttemptId().getApplicationId(); @@ -328,6 +322,8 @@ private static ContainerEntity createContainerEntity( ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); + entity.setIdPrefix(TimelineServiceHelper.invertLong( + containerId.getContainerId())); Identifier parentIdentifier = new Identifier(); parentIdentifier .setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java index cf9ede0..050d677d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.junit.Assert; import org.junit.Test; import org.junit.After; @@ -140,6 +141,8 @@ private Context createMockContext() { entity.getInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO)); Assert.assertEquals(exitStatus, entity.getInfo().get(ContainerMetricsConstants.EXIT_STATUS_INFO)); + Assert.assertEquals(TimelineServiceHelper.invertLong( + cId.getContainerId()), entity.getIdPrefix()); } @Test public void testContainerResourceUsage() { @@ -148,26 +151,28 @@ private Context createMockContext() { Container aContainer = mock(Container.class); when(aContainer.getContainerId()).thenReturn(ContainerId .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L)); + long idPrefix = TimelineServiceHelper.invertLong( + aContainer.getContainerId().getContainerId()); publisher.reportContainerResourceUsage(aContainer, 1024L, 8F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, (float) ResourceCalculatorProcessTree.UNAVAILABLE); verifyPublishedResourceUsageMetrics(timelineClient, 1024L, - ResourceCalculatorProcessTree.UNAVAILABLE); + ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix); } - private void verifyPublishedResourceUsageMetrics( - DummyTimelineClient timelineClient, long memoryUsage, int cpuUsage) { + private void verifyPublishedResourceUsageMetrics(DummyTimelineClient + timelineClient, long memoryUsage, int cpuUsage, long idPrefix) { TimelineEntity[] entities = null; for (int i = 0; i < 10; i++) { entities = timelineClient.getLastPublishedEntities(); @@ -188,6 +193,7 @@ private void verifyPublishedResourceUsageMetrics( assertNotNull("entities are expected to be published", entities); assertEquals("Expected number of metrics notpublished", numberOfResourceMetrics, entities[0].getMetrics().size()); + assertEquals(idPrefix, entities[0].getIdPrefix()); Iterator metrics = entities[0].getMetrics().iterator(); while (metrics.hasNext()) { TimelineMetric metric = metrics.next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 71f88a4..1438e25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -458,6 +458,8 @@ private static ContainerEntity createContainerEntity( ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); + entity.setIdPrefix(TimelineServiceHelper.invertLong( + containerId.getContainerId())); entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT .name(), containerId.getApplicationAttemptId().toString())); return entity; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index ca3f47d..5e1a224 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -289,7 +289,8 @@ public void testPublishContainerMetrics() throws Exception { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity(appFile, 2, - ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0); + ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, + TimelineServiceHelper.invertLong(containerId.getContainerId())); } private RMApp createAppAndRegister(ApplicationId appId) { -- 2.7.4 (Apple Git-66)