commit d12233d985eeb14f1a515020f7f3f146176db72a Author: Haohui Mai Date: Mon Feb 3 11:13:28 2014 -0800 YARN-1553. Do not use HttpConfig.isSecure() in YARN diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 87746bb..aa49852 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -277,9 +277,5 @@ /** Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN */ public static final int HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT = 60; - - // HTTP policies to be used in configuration - public static final String HTTP_POLICY_HTTP_ONLY = "HTTP_ONLY"; - public static final String HTTP_POLICY_HTTPS_ONLY = "HTTPS_ONLY"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index d323f76..15008ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -28,7 +28,6 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class HttpConfig { - private static Policy policy; public enum Policy { HTTP_ONLY, HTTPS_ONLY, @@ -52,28 +51,4 @@ public boolean isHttpsEnabled() { return this == HTTPS_ONLY || this == HTTP_AND_HTTPS; } } - - static { - Configuration conf = new Configuration(); - boolean sslEnabled = conf.getBoolean( - CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, - CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT); - policy = sslEnabled ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY; - } - - public static void setPolicy(Policy policy) { - HttpConfig.policy = policy; - } - - public static boolean isSecure() { - return policy == Policy.HTTPS_ONLY; - } - - public static String getSchemePrefix() { - return (isSecure()) ? "https://" : "http://"; - } - - public static String getScheme(Policy policy) { - return policy == Policy.HTTPS_ONLY ? "https://" : "http://"; - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 969e2fa..b25cd5f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.FileOutputCommitter; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.LocalContainerLauncher; @@ -1387,7 +1386,7 @@ public static void main(String[] args) { // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. MRWebAppUtil.initialize(conf); - HttpConfig.setPolicy(HttpConfig.Policy.HTTP_ONLY); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTP_ONLY"); // log the system properties String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf); if (systemPropsToLog != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 2cc2336..2e1a22e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.http.HttpConfig; /** * Stores Job History configuration keys that can be set by administrators of @@ -135,7 +136,7 @@ public static final String MR_HS_HTTP_POLICY = MR_HISTORY_PREFIX + "http.policy"; public static String DEFAULT_MR_HS_HTTP_POLICY = - CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY; + HttpConfig.Policy.HTTP_ONLY.name(); /**The address the history server webapp is on.*/ public static final String MR_HISTORY_WEBAPP_ADDRESS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java index 49a0407..2d453f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java @@ -71,11 +71,13 @@ public static Policy getYARNHttpPolicy() { } public static String getYARNWebappScheme() { - return HttpConfig.getScheme(httpPolicyInYarn); + return httpPolicyInYarn == HttpConfig.Policy.HTTPS_ONLY ? "https://" + : "http://"; } - + public static String getJHSWebappScheme() { - return HttpConfig.getScheme(httpPolicyInJHS); + return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://" + : "http://"; } public static void setJHSWebappURLWithoutScheme(Configuration conf, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java index 92490a5..28d1417 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java @@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** * HistoryViewer is used to parse and view the JobHistory files @@ -231,7 +232,7 @@ private void printAllTaskAttempts(TaskType taskType) { taskList.append("\t"); taskList.append(attempt.getHostname()).append("\t"); taskList.append(attempt.getError()); - String taskLogsUrl = getTaskLogsUrl(attempt); + String taskLogsUrl = getTaskLogsUrl(WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt); taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a"); System.out.println(taskList.toString()); } @@ -446,7 +447,7 @@ private void printFailedAttempts(FilteredJob filteredJob) { * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ - public static String getTaskLogsUrl( + public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") @@ -457,7 +458,7 @@ public static String getTaskLogsUrl( String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); - return HostUtil.getTaskLogUrl(taskTrackerName, + return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java index 0a42bb7..e131fc8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java @@ -20,7 +20,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.http.HttpConfig; @Private @Unstable @@ -33,9 +32,9 @@ * @param taskAttemptID * @return the taskLogUrl */ - public static String getTaskLogUrl(String taskTrackerHostName, + public static String getTaskLogUrl(String scheme, String taskTrackerHostName, String httpPort, String taskAttemptID) { - return (HttpConfig.getSchemePrefix() + taskTrackerHostName + ":" + + return (scheme + taskTrackerHostName + ":" + httpPort + "/tasklog?attemptid=" + taskAttemptID); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java index 9c92bed..1373f3c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState; @@ -121,7 +120,6 @@ protected void serviceInit(Configuration conf) throws Exception { // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); - HttpConfig.setPolicy(MRWebAppUtil.getJHSHttpPolicy()); try { doSecureLogin(conf); } catch(IOException ie) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 0104704..c3226d8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapreduce.MRConfig; @@ -232,7 +231,7 @@ public void run() { getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), - HttpConfig.isSecure())); + WebAppUtils.useHttps(getConfig()))); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index b5a0b1a..8de941a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -100,7 +100,7 @@ private static void verifyAndSetRMHAIdsList(Configuration conf) { StringBuilder setValue = new StringBuilder(); for (String id: ids) { // verify the RM service addresses configurations for every RMIds - for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String prefix : YarnConfiguration.getServiceAddressConfKeys(conf)) { checkAndSetRMRPCAddress(prefix, id, conf); } setValue.append(id); @@ -158,7 +158,7 @@ private static void verifyAndSetConfValue(String prefix, Configuration conf) { } public static void verifyAndSetAllServiceAddresses(Configuration conf) { - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { verifyAndSetConfValue(confKey, conf); } } @@ -236,7 +236,7 @@ static String getRMHAIdsWarningMessage(String ids) { @InterfaceAudience.Private @VisibleForTesting static String getConfKeyForRMInstance(String prefix, Configuration conf) { - if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) { + if (!YarnConfiguration.getServiceAddressConfKeys(conf).contains(prefix)) { return prefix; } else { String RMId = getRMHAId(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8c8ad16..c585b0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -29,7 +29,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; @@ -187,6 +186,8 @@ /** The https address of the RM web application.*/ public static final String RM_WEBAPP_HTTPS_ADDRESS = RM_PREFIX + "webapp.https.address"; + public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false; + public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml"; public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" @@ -362,14 +363,21 @@ "org.apache.hadoop.yarn.LocalConfigurationProvider"; @Private - public static final List RM_SERVICES_ADDRESS_CONF_KEYS = + public static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = Collections.unmodifiableList(Arrays.asList( RM_ADDRESS, RM_SCHEDULER_ADDRESS, RM_ADMIN_ADDRESS, RM_RESOURCE_TRACKER_ADDRESS, - HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS - : RM_WEBAPP_ADDRESS)); + RM_WEBAPP_HTTPS_ADDRESS)); + + private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS = + Collections.unmodifiableList(Arrays.asList( + RM_ADDRESS, + RM_SCHEDULER_ADDRESS, + RM_ADMIN_ADDRESS, + RM_RESOURCE_TRACKER_ADDRESS, + RM_WEBAPP_HTTPS_ADDRESS)); public static final String AUTO_FAILOVER_PREFIX = RM_HA_PREFIX + "automatic-failover."; @@ -1103,9 +1111,9 @@ public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500; public static final String YARN_HTTP_POLICY_KEY = - YARN_PREFIX + "http.policy"; - public static final String YARN_HTTP_POLICY_DEFAULT = - CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY; + YARN_PREFIX + "http.policy"; + public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY + .name(); public YarnConfiguration() { super(); @@ -1118,6 +1126,12 @@ public YarnConfiguration(Configuration conf) { } } + @Private + public static List getServiceAddressConfKeys(Configuration conf) { + return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS + : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP; + } + /** * Get the socket address for name property as a * InetSocketAddress. @@ -1130,7 +1144,7 @@ public YarnConfiguration(Configuration conf) { public InetSocketAddress getSocketAddr( String name, String defaultAddress, int defaultPort) { String address; - if (HAUtil.isHAEnabled(this) && RM_SERVICES_ADDRESS_CONF_KEYS.contains(name)) { + if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); } else { address = get(name, defaultAddress); @@ -1167,7 +1181,13 @@ public static int getRMDefaultPortNumber(String addressPrefix) { throw new HadoopIllegalArgumentException( "Invalid RM RPC address Prefix: " + addressPrefix + ". The valid value should be one of " - + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS); + + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS_HTTP); } } + + public static boolean useHttps(Configuration conf) { + return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf + .get(YARN_HTTP_POLICY_KEY, + YARN_HTTP_POLICY_DEFAULT)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 9fcc2bd..3269b8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -29,7 +29,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; @@ -65,12 +64,17 @@ public TimelineClientImpl() { } protected void serviceInit(Configuration conf) throws Exception { - resURI = new URI(JOINER.join(HttpConfig.getSchemePrefix(), - HttpConfig.isSecure() ? conf.get( - YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS) : conf.get( - YarnConfiguration.AHS_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + if (YarnConfiguration.useHttps(conf)) { + resURI = URI + .create(JOINER.join("https://", conf.get( + YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS), + RESOURCE_URI_STR)); + } else { + resURI = URI.create(JOINER.join("http://", conf.get( + YarnConfiguration.AHS_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + } super.serviceInit(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 317baec..db91212 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -37,7 +37,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -216,9 +218,11 @@ public void setup() { System.exit(1); } } - HttpServer2.Builder builder = new HttpServer2.Builder().setName(name) - .addEndpoint(URI.create("http://" + bindAddress + ":" + port)) - .setConf(conf).setFindPort(findPort) + HttpServer2.Builder builder = new HttpServer2.Builder() + .setName(name) + .addEndpoint( + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + + ":" + port)).setConf(conf).setFindPort(findPort) .setACL(new AdminACLsManager(conf).getAdminAcl()) .setPathSpec(pathList.toArray(new String[0])); @@ -231,6 +235,11 @@ public void setup() { .setKeytabConfKey(spnegoKeytabKey) .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()); } + + if (YarnConfiguration.useHttps(conf)) { + WebAppUtils.loadSslConfiguration(builder, conf); + } + HttpServer2 server = builder.build(); for(ServletStruct struct: servlets) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 4a288c4..93242a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -26,20 +26,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; -import com.google.common.base.Joiner; - @Private @Evolving public class WebAppUtils { - private static final Joiner JOINER = Joiner.on(""); - public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); hostname = @@ -51,7 +47,7 @@ public static void setRMWebAppPort(Configuration conf, int port) { public static void setRMWebAppHostnameAndPort(Configuration conf, String hostname, int port) { String resolvedAddress = hostname + ":" + port; - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); } else { conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); @@ -60,7 +56,7 @@ public static void setRMWebAppHostnameAndPort(Configuration conf, public static void setNMWebAppHostNameAndPort(Configuration conf, String hostName, int port) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostName + ":" + port); } else { @@ -70,16 +66,11 @@ public static void setNMWebAppHostNameAndPort(Configuration conf, } public static String getRMWebAppURLWithScheme(Configuration conf) { - return JOINER.join(HttpConfig.getSchemePrefix(), - HttpConfig.isSecure() ? conf.get( - YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS) : conf.get( - YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); + return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(conf); } public static String getRMWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS); }else { @@ -97,13 +88,13 @@ public static String getProxyHostAndPort(Configuration conf) { } public static String getResolvedRMWebAppURLWithScheme(Configuration conf) { - return HttpConfig.getSchemePrefix() + return getHttpSchemePrefix(conf) + getResolvedRMWebAppURLWithoutScheme(conf); } public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) { return getResolvedRMWebAppURLWithoutScheme(conf, - HttpConfig.isSecure() ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY); + YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY); } public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf, @@ -140,7 +131,7 @@ public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf, } public static String getNMWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS); } else { @@ -150,7 +141,7 @@ public static String getNMWebAppURLWithoutScheme(Configuration conf) { } public static String getAHSWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS); } else { @@ -177,8 +168,39 @@ public static String getURLWithScheme(String schemePrefix, String url) { public static String getLogUrl(String nodeHttpAddress, String allocatedNode, ContainerId containerId, String user) { - return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/", + return join("//", nodeHttpAddress, "/logs", "/", allocatedNode, "/", ConverterUtils.toString(containerId), "/", ConverterUtils.toString(containerId), "/", user); } + + public static boolean useHttps(Configuration conf) { + return YarnConfiguration.useHttps(conf); + } + + /** + * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on + * the configuration. + * + * @return the schmeme (HTTP / HTTPS) + */ + public static String getHttpSchemePrefix(Configuration conf) { + return YarnConfiguration.useHttps(conf) ? "https://" : "http://"; + } + + public static HttpServer2.Builder loadSslConfiguration( + HttpServer2.Builder builder, Configuration conf) { + Configuration sslConf = new Configuration(false); + boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT; + sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT); + + return builder + .needsClientAuth(needsClientAuth) + .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) + .keyStore(sslConf.get("ssl.server.keystore.location"), + sslConf.get("ssl.server.keystore.password"), + sslConf.get("ssl.server.keystore.type", "jks")) + .trustStore(sslConf.get("ssl.server.truststore.location"), + sslConf.get("ssl.server.truststore.password"), + sslConf.get("ssl.server.truststore.type", "jks")); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index 891b434..6ced5f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -54,7 +54,7 @@ public void setUp() { conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { // configuration key itself cannot contains space/tab/return chars. conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); @@ -95,7 +95,7 @@ public void testVerifyAndSetConfiguration() throws Exception { StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf)); assertEquals("Should be saved as Trimmed string", RM1_NODE_ID, HAUtil.getRMHAId(conf)); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { assertEquals("RPC address not set for " + confKey, RM1_ADDRESS, conf.get(confKey)); } @@ -117,7 +117,7 @@ public void testVerifyAndSetConfiguration() throws Exception { // simulate the case YarnConfiguration.RM_HA_ID is not set conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } @@ -134,7 +134,7 @@ public void testVerifyAndSetConfiguration() throws Exception { conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID); conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + RM1_NODE_ID); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { // simulate xml with invalid node id conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID); } @@ -169,7 +169,7 @@ public void testVerifyAndSetConfiguration() throws Exception { conf.clear(); conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 9a15fe7..09bb958 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -163,7 +163,7 @@ protected void render(Block html) { .append(startTime) .append("\",\"") .append( nodeLink == null ? "N/A" : StringEscapeUtils diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index a169c12..d9a5408 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -28,8 +28,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; @@ -397,15 +395,8 @@ public static void main(String[] args) { StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); NodeManager nodeManager = new NodeManager(); Configuration conf = new YarnConfiguration(); - setHttpPolicy(conf); nodeManager.initAndStartNodeManager(conf, false); } - - private static void setHttpPolicy(Configuration conf) { - HttpConfig.setPolicy(Policy.fromString(conf.get( - YarnConfiguration.YARN_HTTP_POLICY_KEY, - YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); - } @VisibleForTesting @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java index 424da04..c198ae6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 1040cc5..9d39119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1011,7 +1011,6 @@ public static void main(String argv[]) { ShutdownHookManager.get().addShutdownHook( new CompositeServiceShutdownHook(resourceManager), SHUTDOWN_HOOK_PRIORITY); - setHttpPolicy(conf); resourceManager.init(conf); resourceManager.start(); } catch (Throwable t) { @@ -1019,12 +1018,6 @@ public static void main(String argv[]) { System.exit(-1); } } - - private static void setHttpPolicy(Configuration conf) { - HttpConfig.setPolicy(Policy.fromString(conf.get( - YarnConfiguration.YARN_HTTP_POLICY_KEY, - YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); - } /** * Register the handlers for alwaysOn services diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index ce246db..eaddddf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -502,10 +502,11 @@ private String generateProxyUriWithScheme( final String trackingUriWithoutScheme) { this.readLock.lock(); try { + final String scheme = WebAppUtils.getHttpSchemePrefix(conf); URI trackingUri = StringUtils.isEmpty(trackingUriWithoutScheme) ? null : - ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); + ProxyUriUtils.getUriFromAMUrl(scheme, trackingUriWithoutScheme); String proxy = WebAppUtils.getProxyHostAndPort(conf); - URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy); URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, applicationAttemptId.getApplicationId()); return result.toASCIIString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index 445a5a2..ac8578e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -27,7 +27,7 @@ import java.util.Collection; -import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -55,13 +56,16 @@ private ApplicationACLsManager aclsManager; private QueueACLsManager queueACLsManager; + private final Configuration conf; @Inject AppBlock(ResourceManager rm, ViewContext ctx, - ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager) { + ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager, + Configuration conf) { super(ctx); this.aclsManager = aclsManager; this.queueACLsManager = queueACLsManager; + this.conf = conf; } @Override @@ -86,7 +90,7 @@ protected void render(Block html) { puts("Application not found: "+ aid); return; } - AppInfo app = new AppInfo(rmApp, true); + AppInfo app = new AppInfo(rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); // Check for the authorization. String remoteUser = request().getRemoteUser(); @@ -146,7 +150,7 @@ protected void render(Block html) { table.tr((odd = !odd) ? _ODD : _EVEN). td(String.valueOf(attemptInfo.getAttemptId())). td(Times.format(attemptInfo.getStartTime())). - td().a(".nodelink", url(HttpConfig.getSchemePrefix(), + td().a(".nodelink", url("//", attemptInfo.getNodeHttpAddress()), attemptInfo.getNodeHttpAddress())._(). td().a(".logslink", url(attemptInfo.getLogsLink()), "logs")._(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index cec95ac..4f644d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -36,16 +37,19 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; class AppsBlock extends HtmlBlock { final ConcurrentMap apps; + private final Configuration conf; -@Inject AppsBlock(RMContext rmContext, ViewContext ctx) { +@Inject AppsBlock(RMContext rmContext, ViewContext ctx, Configuration conf) { super(ctx); apps = rmContext.getRMApps(); + this.conf = conf; } @Override public void render(Block html) { @@ -79,7 +83,7 @@ if (reqAppStates != null && !reqAppStates.contains(app.createApplicationState())) { continue; } - AppInfo appInfo = new AppInfo(app, true); + AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf)); String percent = String.format("%.1f", appInfo.getProgress()); //AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js appsTableData.append("[\" trackingUriPlugins; private final String rmAppPageUrlBase; + private final String AMUrlScheme; private static class _ implements Hamlet._ { //Empty @@ -96,6 +97,7 @@ public WebAppProxyServlet() TrackingUriPlugin.class); this.rmAppPageUrlBase = StringHelper.pjoin( WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app"); + this.AMUrlScheme = WebAppUtils.getHttpSchemePrefix(conf); } /** @@ -300,7 +302,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) return; } else { if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { - trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original); + trackingUri = ProxyUriUtils.getUriFromAMUrl(AMUrlScheme, original); } else { trackingUri = new URI(original); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index 7f81f9b..e35ed84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.http.FilterInitializer; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -39,7 +38,7 @@ public void initFilter(FilterContainer container, Configuration conf) { String[] parts = proxy.split(":"); params.put(AmIpFilter.PROXY_HOST, parts[0]); params.put(AmIpFilter.PROXY_URI_BASE, - HttpConfig.getSchemePrefix() + proxy + + WebAppUtils.getHttpSchemePrefix(conf) + proxy + System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV)); container.addFilter(FILTER_NAME, FILTER_CLASS, params); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index f39ab3e..c53d098 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -288,8 +288,9 @@ public void start() { YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); proxyServer = new HttpServer2.Builder() .setName("proxy") - .addEndpoint(URI.create("http://" + bindAddress + ":0")) - .setFindPort(true) + .addEndpoint( + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + + ":0")).setFindPort(true) .setConf(conf) .setACL(acl) .build();