diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index a955eaa..b6c0a24 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -18,8 +18,15 @@ package org.apache.hadoop.mapreduce.v2.jobhistory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.net.NetUtils; /** * Stores Job History configuration keys that can be set by administrators of @@ -130,6 +137,13 @@ public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT; + /**The https address the history server webapp is on.*/ + public static final String MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + MR_HISTORY_PREFIX + "webapp.https.address"; + public static final int DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT = 19890; + public static final String DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT; + /**The kerberos principal to be used for spnego filter for history server*/ public static final String MR_WEBAPP_SPNEGO_USER_NAME_KEY = MR_HISTORY_PREFIX + "webapp.spnego-principal"; @@ -160,4 +174,35 @@ */ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false; + public static String getMRHistoryWebAppHostAndPort(Configuration conf) { + InetSocketAddress address = null; + if (HttpConfig.isSecure()) { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT); + } else { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); } + address = NetUtils.getConnectAddress(address); + StringBuffer sb = new StringBuffer(); + InetAddress resolved = address.getAddress(); + if (resolved == null || resolved.isAnyLocalAddress() || + resolved.isLoopbackAddress()) { + String lh = address.getHostName(); + try { + lh = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + //Ignore and fallback. + } + sb.append(lh); + } else { + sb.append(address.getHostName()); + } + sb.append(":").append(address.getPort()); + return sb.toString(); + } + } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index ab4bfa7..a8d78d3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapreduce.MRConfig; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; +import org.hamcrest.core.IsEqual; /** * Configures and starts the MR-specific components in the YARN cluster. @@ -147,6 +149,7 @@ public JobHistoryServerWrapper() { @Override public synchronized void serviceStart() throws Exception { + boolean isSecure = HttpConfig.isSecure(); try { if (!getConfig().getBoolean( JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, @@ -155,8 +158,13 @@ public synchronized void serviceStart() throws Exception { // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - hostname + ":0"); + if (isSecure) { + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + hostname + ":0"); + } else { + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + hostname + ":0"); + } getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } @@ -182,17 +190,24 @@ public void run() { //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + if (isSecure) { + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + historyServer.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS)); + } else { + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + historyServer.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + } LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + - getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); + YarnConfiguration.getRMWebAppHostAndPort(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + - getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + JHAdminConfig.getMRHistoryWebAppHostAndPort(getConfig())); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index cc2f38b..02b0eda 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -163,6 +163,14 @@ public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_RM_WEBAPP_PORT; + /** The https address of the RM web application.*/ + public static final String RM_WEBAPP_HTTPS_ADDRESS = + RM_PREFIX + "webapp.https.address"; + + public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; + public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + + DEFAULT_RM_WEBAPP_HTTPS_PORT; + public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; @@ -520,6 +528,13 @@ public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_NM_WEBAPP_PORT; + /** NM Webapp https address.**/ + public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX + + "webapp.https.address"; + public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044; + public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + + DEFAULT_NM_WEBAPP_HTTPS_PORT; + /** How often to monitor containers.*/ public final static String NM_CONTAINER_MON_INTERVAL_MS = NM_PREFIX + "container-monitor.interval-ms"; @@ -815,10 +830,18 @@ public static String getProxyHostAndPort(Configuration conf) { } public static String getRMWebAppHostAndPort(Configuration conf) { - InetSocketAddress address = conf.getSocketAddr( - YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + InetSocketAddress address = null; + if (HttpConfig.isSecure()) { + address = + conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); + } else { + address = + conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + } address = NetUtils.getConnectAddress(address); StringBuffer sb = new StringBuffer(); InetAddress resolved = address.getAddress(); @@ -840,7 +863,11 @@ public static String getRMWebAppHostAndPort(Configuration conf) { public static String getRMWebAppURL(Configuration conf) { return JOINER.join(HttpConfig.getSchemePrefix(), - getRMWebAppHostAndPort(conf)); + HttpConfig.isSecure() ? conf.get( + YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS) : conf.get( + YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java index fab88b9..07feb58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java @@ -22,7 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -54,8 +54,15 @@ public WebServer(Context nmContext, ResourceView resourceView, @Override protected void serviceStart() throws Exception { - String bindAddress = getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS, + String bindAddress = null; + + if (HttpConfig.isSecure()) { + bindAddress = getConfig().get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS); + } else { + bindAddress = getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS); + } LOG.info("Instantiating NMWebApp at " + bindAddress); try { this.webApp = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index ff57fc5..bce98ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; @@ -441,12 +442,16 @@ protected void serviceStart() throws Exception { } if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { - String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); + boolean isSecure = HttpConfig.isSecure(); + String hostname = YarnConfiguration.getRMWebAppHostAndPort(getConfig()); hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; int port = webApp.port(); String resolvedAddress = hostname + ":" + port; - conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); + if (isSecure) { + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); + } else { + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); + } } super.serviceStart(); @@ -695,6 +700,16 @@ public void handle(RMNodeEvent event) { } protected void startWepApp() { + String rmWebAppAddress = null; + if (HttpConfig.isSecure()) { + rmWebAppAddress = + conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS); + } else { + rmWebAppAddress = + conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); + } Builder builder = WebApps .$for("cluster", ApplicationMasterService.class, masterService, @@ -704,8 +719,7 @@ protected void startWepApp() { YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); + .at(rmWebAppAddress); String proxyHostAndPort = YarnConfiguration.getProxyHostAndPort(conf); if(YarnConfiguration.getRMWebAppHostAndPort(conf). equals(proxyHostAndPort)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 1281c24..36fb835 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; @@ -39,7 +40,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.server.api.ResourceTracker; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; @@ -204,8 +204,9 @@ public synchronized void serviceStart() throws Exception { hostname + ":0"); getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0"); - getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS, - hostname + ":0"); + getConfig().set(HttpConfig.isSecure()? + YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS: + YarnConfiguration.RM_WEBAPP_ADDRESS, hostname + ":0"); } resourceManager = new ResourceManager() { @Override @@ -238,7 +239,7 @@ public void run() { LOG.info("MiniYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniYARN ResourceManager web address: " + - getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); + YarnConfiguration.getRMWebAppHostAndPort(getConfig())); } @Override @@ -317,8 +318,13 @@ public synchronized void serviceStart() throws Exception { MiniYARNCluster.getHostname() + ":0"); getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, MiniYARNCluster.getHostname() + ":0"); - getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, - MiniYARNCluster.getHostname() + ":0"); + if (HttpConfig.isSecure()) { + getConfig().set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + } else { + getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + } // Disable resource checks by default if (!getConfig().getBoolean(