diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index ae5efa5..1fef76e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -56,6 +56,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 39f17d8..fab131c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -19,25 +19,32 @@ package org.apache.hadoop.yarn.server.timelineservice; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.fail; - public class TestTimelineServiceClientIntegration { + private static TimelineCollectorManager collectorManager; private static PerNodeTimelineCollectorsAuxService auxService; @BeforeClass public static void setupClass() throws Exception { try { - auxService = PerNodeTimelineCollectorsAuxService.launchServer(new String[0]); + collectorManager = new MyTimelineCollectorManager(); + auxService = + PerNodeTimelineCollectorsAuxService.launchServer(new String[0], + collectorManager); auxService.addApplication(ApplicationId.newInstance(0, 1)); } catch (ExitUtil.ExitException e) { fail(); @@ -56,6 +63,9 @@ public void testPutEntities() throws Exception { TimelineClient client = TimelineClient.createTimelineClient(ApplicationId.newInstance(0, 1)); try { + // set the timeline service address manually + client.setTimelineServiceAddress( + collectorManager.getRestServerBindAddress()); client.init(new YarnConfiguration()); client.start(); TimelineEntity entity = new TimelineEntity(); @@ -63,10 +73,20 @@ public void testPutEntities() throws Exception { entity.setId("test entity id"); client.putEntities(entity); client.putEntitiesAsync(entity); - } catch(Exception e) { - fail(); } finally { client.stop(); } } + + private static class MyTimelineCollectorManager extends + TimelineCollectorManager { + public MyTimelineCollectorManager() { + super(); + } + + @Override + protected CollectorNodemanagerProtocol getNMCollectorService() { + return mock(CollectorNodemanagerProtocol.class); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java index f5dbd03..59ecef1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java @@ -174,14 +174,16 @@ public ByteBuffer getMetaData() { @VisibleForTesting public static PerNodeTimelineCollectorsAuxService - launchServer(String[] args) { + launchServer(String[] args, TimelineCollectorManager collectorManager) { Thread .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage( PerNodeTimelineCollectorsAuxService.class, args, LOG); PerNodeTimelineCollectorsAuxService auxService = null; try { - auxService = new PerNodeTimelineCollectorsAuxService(); + auxService = collectorManager == null ? + new PerNodeTimelineCollectorsAuxService() : + new PerNodeTimelineCollectorsAuxService(collectorManager); ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService), SHUTDOWN_HOOK_PRIORITY); YarnConfiguration conf = new YarnConfiguration(); @@ -207,6 +209,6 @@ public void run() { } public static void main(String[] args) { - launchServer(args); + launchServer(args, null); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java index deb54b2..3691162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import com.google.common.annotations.VisibleForTesting; + /** * Class that manages adding and removing collectors and their lifecycle. It * provides thread safety access to the collectors inside. @@ -83,7 +85,8 @@ static TimelineCollectorManager getInstance() { return INSTANCE; } - TimelineCollectorManager() { + @VisibleForTesting + protected TimelineCollectorManager() { super(TimelineCollectorManager.class.getName()); } @@ -257,8 +260,8 @@ private void reportNewCollectorToNM(ApplicationId appId) nmCollectorService.reportNewCollectorInfo(request); } - // protected for test - protected CollectorNodemanagerProtocol getNMCollectorService(){ + @VisibleForTesting + protected CollectorNodemanagerProtocol getNMCollectorService() { Configuration conf = getConfig(); final YarnRPC rpc = YarnRPC.create(conf); @@ -268,4 +271,8 @@ protected CollectorNodemanagerProtocol getNMCollectorService(){ nmCollectorServiceAddress, conf); } + @VisibleForTesting + public String getRestServerBindAddress() { + return timelineRestServerBindAddress; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index a270c4d..3b20352 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; import org.junit.Test; @@ -111,7 +112,8 @@ public void testLaunch() throws Exception { PerNodeTimelineCollectorsAuxService auxService = null; try { auxService = - PerNodeTimelineCollectorsAuxService.launchServer(new String[0]); + PerNodeTimelineCollectorsAuxService.launchServer(new String[0], + createCollectorManager()); } catch (ExitUtil.ExitException e) { assertEquals(0, e.status); ExitUtil.resetFirstExitException(); @@ -136,14 +138,22 @@ public void testLaunch() throws Exception { } private PerNodeTimelineCollectorsAuxService createCollector() { - TimelineCollectorManager - collectorManager = spy(new TimelineCollectorManager()); - doReturn(new Configuration()).when(collectorManager).getConfig(); + TimelineCollectorManager collectorManager = createCollectorManager(); PerNodeTimelineCollectorsAuxService auxService = spy(new PerNodeTimelineCollectorsAuxService(collectorManager)); return auxService; } + private TimelineCollectorManager createCollectorManager() { + TimelineCollectorManager collectorManager = + spy(new TimelineCollectorManager()); + doReturn(new Configuration()).when(collectorManager).getConfig(); + CollectorNodemanagerProtocol nmCollectorService = + mock(CollectorNodemanagerProtocol.class); + doReturn(nmCollectorService).when(collectorManager).getNMCollectorService(); + return collectorManager; + } + private ContainerId getAMContainerId() { return getContainerId(1L); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java index b77429c..541665b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import java.util.ArrayList; @@ -32,15 +33,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.junit.Test; public class TestTimelineCollectorManager { @Test(timeout=60000) public void testMultithreadedAdd() throws Exception { - final TimelineCollectorManager collectorManager = - spy(new TimelineCollectorManager()); - doReturn(new Configuration()).when(collectorManager).getConfig(); + final TimelineCollectorManager collectorManager = createCollectorManager(); final int NUM_APPS = 5; List> tasks = new ArrayList>(); @@ -66,15 +66,14 @@ public Boolean call() { } // check the keys for (int i = 0; i < NUM_APPS; i++) { - assertTrue(collectorManager.containsKey(String.valueOf(i))); + final ApplicationId appId = ApplicationId.newInstance(0L, i); + assertTrue(collectorManager.containsKey(appId.toString())); } } @Test public void testMultithreadedAddAndRemove() throws Exception { - final TimelineCollectorManager collectorManager = - spy(new TimelineCollectorManager()); - doReturn(new Configuration()).when(collectorManager).getConfig(); + final TimelineCollectorManager collectorManager = createCollectorManager(); final int NUM_APPS = 5; List> tasks = new ArrayList>(); @@ -102,7 +101,18 @@ public Boolean call() { } // check the keys for (int i = 0; i < NUM_APPS; i++) { - assertFalse(collectorManager.containsKey(String.valueOf(i))); + final ApplicationId appId = ApplicationId.newInstance(0L, i); + assertFalse(collectorManager.containsKey(appId.toString())); } } + + private TimelineCollectorManager createCollectorManager() { + final TimelineCollectorManager collectorManager = + spy(new TimelineCollectorManager()); + doReturn(new Configuration()).when(collectorManager).getConfig(); + CollectorNodemanagerProtocol nmCollectorService = + mock(CollectorNodemanagerProtocol.class); + doReturn(nmCollectorService).when(collectorManager).getNMCollectorService(); + return collectorManager; + } }