diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 05c6cbf..b13a916 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1311,6 +1311,10 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + // TODO remove this const after we finish YARN-3039 + @Private + public static final String _TEMP_TIMELINE_SERVICE_V2_WEBAPP_ADDRESS = "0.0.0.0:8191"; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 46b5850..9b19c7e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -62,6 +62,10 @@ protected MiniYARNCluster yarnCluster = null; protected YarnConfiguration conf = null; private static final int NUM_NMS = 1; + private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_aggregator"; + private static final String TIMELINE_AUX_SERVICE_CLASS = + "org.apache.hadoop.yarn.server.timelineservice.aggregator" + + ".PerNodeAggregatorServer"; protected final static String APPMASTER_JAR = JarFinder.getJar(ApplicationMaster.class); @@ -75,12 +79,16 @@ protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); + conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME); + conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + + ".class", TIMELINE_AUX_SERVICE_CLASS); if (yarnCluster == null) { yarnCluster = @@ -140,15 +148,21 @@ public void tearDown() throws IOException { @Test(timeout=90000) public void testDSShellWithDomain() throws Exception { - testDSShell(true); + testDSShell(true, "v1"); } @Test(timeout=90000) public void testDSShellWithoutDomain() throws Exception { - testDSShell(false); + testDSShell(false, "v1"); + } + + @Test(timeout=90000) + public void testDSShellWithoutDomainV2() throws Exception { + testDSShell(false, "v2"); } - public void testDSShell(boolean haveDomain) throws Exception { + public void testDSShell(boolean haveDomain, String timelineVersion) + throws Exception { String[] args = { "--jar", APPMASTER_JAR, @@ -175,9 +189,17 @@ public void testDSShell(boolean haveDomain) throws Exception { "writer_user writer_group", "--create" }; - List argsList = new ArrayList(Arrays.asList(args)); - argsList.addAll(Arrays.asList(domainArgs)); - args = argsList.toArray(new String[argsList.size()]); + args = mergeArgs(args, domainArgs); + } + boolean isTestingTimelineV2 = false; + if (timelineVersion.equalsIgnoreCase("v2")) { + String[] timelineArgs = { + "--timeline_service_version", + "v2" + }; + isTestingTimelineV2 = true; + args = mergeArgs(args, timelineArgs); + LOG.info("Setup: Using timeline v2!"); } LOG.info("Initializing DS Client"); @@ -231,7 +253,15 @@ public void run() { LOG.info("Client run completed. Result=" + result); Assert.assertTrue(result.get()); - TimelineDomain domain = null; + if (!isTestingTimelineV2) { + checkTimelineV1(haveDomain); + } else { + checkTimelineV2(haveDomain); + } + } + + private void checkTimelineV1(boolean haveDomain) throws Exception { + TimelineDomain domain = null; if (haveDomain) { domain = yarnCluster.getApplicationHistoryServer() .getTimelineStore().getDomain("TEST_DOMAIN"); @@ -275,6 +305,24 @@ public void run() { } } + private void checkTimelineV2(boolean haveDomain) { + // TODO check timeline V2 here after we have a storage layer + } + + /** + * Utility function to merge two String arrays to form a new String array for + * our argumemts. + * + * @param args + * @param newArgs + * @return a String array consists of {args, newArgs} + */ + private String[] mergeArgs(String[] args, String[] newArgs) { + List argsList = new ArrayList(Arrays.asList(args)); + argsList.addAll(Arrays.asList(newArgs)); + return argsList.toArray(new String[argsList.size()]); + } + /* * NetUtils.getHostname() returns a string in the form "hostname/ip". * Sometimes the hostname we get is the FQDN and sometimes the short name. In diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index d5faaac..82f3c8a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -282,9 +282,14 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS); } else { - timelineServiceAddress = conf.get( + // Use a temporary webapp address for timeline v2 for now. + // TODO change this after we finish YARN-3039 service discovery + timelineServiceAddress = + YarnConfiguration._TEMP_TIMELINE_SERVICE_V2_WEBAPP_ADDRESS; + // For v1 + /* timelineServiceAddress = YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS); + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);*/ } LOG.info("Timeline service address: " + timelineServiceAddress); super.serviceInit(conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/BaseAggregatorService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/BaseAggregatorService.java index 46e5574..e362139 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/BaseAggregatorService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/BaseAggregatorService.java @@ -72,6 +72,12 @@ protected void serviceStop() throws Exception { */ public void postEntities(TimelineEntities entities, UserGroupInformation callerUgi) { + // Add this output temporarily for our prototype + // TODO remove this after we have an actual implementation + LOG.info("SUCCESS - TIMELINE V2 PROTOTYPE"); + LOG.info("postEntities(entities=" + entities + ", callerUgi=" + + callerUgi + ")"); + // TODO implement if (LOG.isDebugEnabled()) { LOG.debug("postEntities(entities=" + entities + ", callerUgi=" + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeAggregatorServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeAggregatorServer.java index 55c6271..a9f7161 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeAggregatorServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeAggregatorServer.java @@ -104,11 +104,9 @@ protected void serviceStop() throws Exception { private void startWebApp() { Configuration conf = getConfig(); - // use the same ports as the old ATS for now; we could create new properties - // for the new timeline service if needed - String bindAddress = WebAppUtils.getWebAppBindURL(conf, - YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, - WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); + // Use a temporary webapp address for timeline v2 for now. + // TODO change this after we finish YARN-3039 service discovery + String bindAddress = YarnConfiguration._TEMP_TIMELINE_SERVICE_V2_WEBAPP_ADDRESS; LOG.info("Instantiating the per-node aggregator webapp at " + bindAddress); try { Configuration confForInfoServer = new Configuration(conf);