diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 04c84ca..9924683 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -254,6 +255,7 @@ public TimelineClientImpl() { super(TimelineClientImpl.class.getName()); } + @Override protected void serviceInit(Configuration conf) throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation realUgi = ugi.getRealUser(); @@ -281,20 +283,19 @@ protected void serviceInit(Configuration conf) throws Exception { TimelineJerseyRetryFilter retryFilter = new TimelineJerseyRetryFilter(); client.addFilter(retryFilter); - if (YarnConfiguration.useHttps(conf)) { - resURI = URI - .create(JOINER.join("https://", conf.get( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS), - RESOURCE_URI_STR)); - } else { - resURI = URI.create(JOINER.join("http://", conf.get( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + // Create the URL as late as possible to give an opportunity for config to + // be updated with the right service address. + resURI = + URI.create(JOINER.join( + WebAppUtils.getTimelineWebAppURLWithScheme(getConfig()), RESOURCE_URI_STR)); - } LOG.info("Timeline service address: " + resURI); - super.serviceInit(conf); + super.serviceStart(); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 459c110..e37e88b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -32,8 +32,8 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.HAUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.RMHAUtils; @@ -262,6 +262,10 @@ public static String getNMWebAppURLWithoutScheme(Configuration conf) { } } + public static String getTimelineWebAppURLWithScheme(Configuration conf) { + return getHttpSchemePrefix(conf) + getAHSWebAppURLWithoutScheme(conf); + } + public static String getAHSWebAppURLWithoutScheme(Configuration conf) { if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 0f09735..5c4c13f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index aa796ed..39ac74f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -133,6 +133,9 @@ * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager * @param enableAHS enable ApplicationHistoryServer or not + * @deprecated Use other constructors instead and use configuration to + * enable or disable Timeline Service / History Service instead of + * the parameter enableAHS. */ @Deprecated public MiniYARNCluster( @@ -248,6 +251,13 @@ public void serviceInit(Configuration conf) throws Exception { rmIds = rmIdsCollection.toArray(new String[rmIdsCollection.size()]); } + + // Timeline Service should start first as RM depends on it. + if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { + addService(new ApplicationHistoryServerWrapper()); + } + for (int i = 0; i < resourceManagers.length; i++) { resourceManagers[i] = createResourceManager(); if (!useFixedPorts) { @@ -264,11 +274,6 @@ public void serviceInit(Configuration conf) throws Exception { useRpc ? new CustomNodeManager() : new ShortCircuitedNodeManager(); addService(new NodeManagerWrapper(index)); } - - if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { - addService(new ApplicationHistoryServerWrapper()); - } super.serviceInit( conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java index e67a236..3edf848 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -18,7 +18,13 @@ package org.apache.hadoop.yarn.server; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil; @@ -28,9 +34,39 @@ public class TestMiniYarnCluster { @Test + public void testDaemonOrder() { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + + MiniYARNCluster cluster = + new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), 1, 1, 1, + 1); + cluster.init(conf); + + List services = cluster.getServices(); + + assertEquals("Number of registered services ", 3, services.size()); + + assertTrue("Service 0 is " + services.get(0).getName() + + " not Timeline Service!", + services.get(0).getName().contains("ApplicationHistoryServerWrapper")); + assertTrue("Service 1 is " + services.get(1).getName() + + " and not ResourceManager_0!", + services.get(1).getName().contains("ResourceManagerWrapper_0")); + assertTrue("Service 2 is " + services.get(2).getName() + + " not NodeManager!", + services.get(2).getName().contains("NodeManagerWrapper_0")); + } + + @Test + @SuppressWarnings("deprecation") public void testTimelineServiceStartInMiniCluster() throws Exception { + Configuration conf = new YarnConfiguration(); - int numNodeManagers = 1; + + // No need for NodeManagers in this test at all. + int numNodeManagers = 0; + int numLocalDirs = 1; int numLogDirs = 1; boolean enableAHS;