diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index d906d96..ee4cb47 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -39,7 +39,7 @@ } @VisibleForTesting - static void setSecure(boolean secure) { + public static void setSecure(boolean secure) { sslEnabled = secure; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java index 181fd37..ccbdb83 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -112,6 +113,13 @@ public MRClientService(AppContext appContext) { protected void serviceStart() throws Exception { Configuration conf = getConfig(); + // Explicitly disabling SSL for map reduce task as we can't allow MR users + // to gain access to keystore file for opening SSL listener. We can trust + // RM/NM to issue SSL certificates but definitely not MR-AM as it is + // running in user-land. + conf.set(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, "false"); + HttpConfig.setSecure(false); + YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = new InetSocketAddress(0); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java index 7ce4d5d..db6838f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -148,7 +149,13 @@ protected void register() { if (serviceAddr != null) { request.setHost(serviceAddr.getHostName()); request.setRpcPort(serviceAddr.getPort()); - request.setTrackingUrl(serviceAddr.getHostName() + ":" + clientService.getHttpPort()); + String scheme = "http://"; + if (getConfig().getBoolean(MRConfig.SSL_ENABLED_KEY, + MRConfig.SSL_ENABLED_KEY_DEFAULT)) { + scheme = "https://"; + } + request.setTrackingUrl(scheme + serviceAddr.getHostName() + ":" + + clientService.getHttpPort()); } RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java index bbac5fc..879f70d 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java @@ -84,6 +84,11 @@ "mapreduce.shuffle.ssl.enabled"; public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; + + public static final String SSL_ENABLED_KEY = + "mapreduce.ssl.enabled"; + + public static final boolean SSL_ENABLED_KEY_DEFAULT = false; public static final String SHUFFLE_CONSUMER_PLUGIN = "mapreduce.job.reduce.shuffle.consumer.plugin.class"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java index aee2cc8..9f2afd2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java @@ -143,8 +143,37 @@ public static URI getProxyUri(URI originalUri, URI proxyUri, */ public static URI getUriFromAMUrl(String noSchemeUrl) throws URISyntaxException { - return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl); + if (getSchemeFromUrl(noSchemeUrl).isEmpty()) { + /* + * check is made to make sure if AM reports with scheme then it will be + * used by default otherwise it will default to the one configured using + * "hadoop.ssl.enabled". + */ + return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl); + } else { + return new URI(noSchemeUrl); + } } + + /** + * Create a URI form a no scheme Url, such as is returned by the AM. + * @param noSchemeUrl the URL formate returned by an AM + * @return a URI with an http scheme + * @throws URISyntaxException if the url is not formatted correctly. + */ + public static URI getUriFromAMUrl(String scheme, String noSchemeUrl) + throws URISyntaxException { + if (getSchemeFromUrl(noSchemeUrl).isEmpty()) { + /* + * check is made to make sure if AM reports with scheme then it will be + * used by default otherwise it will default to the one configured using + * "hadoop.ssl.enabled". + */ + return new URI(scheme + "://" + noSchemeUrl); + } else { + return new URI(noSchemeUrl); + } + } /** * Returns the first valid tracking link, if any, from the given id from the @@ -169,4 +198,20 @@ public static URI getUriFromTrackingPlugins(ApplicationId id, } return null; } + + /** + * Returns the scheme if present in the url + * eg. "https://issues.apache.org/jira/browse/YARN" > "https" + */ + public static String getSchemeFromUrl(String url) { + int index = 0; + if (url != null) { + index = url.indexOf("://"); + } + if (index > 0) { + return url.substring(0, index); + } else { + return ""; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 5fd426c..99a480b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -163,7 +163,6 @@ private static void proxyLink(HttpServletRequest req, } config.setLocalAddress(localAddress); HttpMethod method = new GetMethod(uri.getEscapedURI()); - @SuppressWarnings("unchecked") Enumeration names = req.getHeaderNames(); while(names.hasMoreElements()) { @@ -293,14 +292,17 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) } String original = applicationReport.getOriginalTrackingUrl(); URI trackingUri = null; - if (original != null) { - trackingUri = ProxyUriUtils.getUriFromAMUrl(original); - } // fallback to ResourceManager's app page if no tracking URI provided if(original == null || original.equals("N/A")) { resp.sendRedirect(resp.encodeRedirectURL( StringHelper.pjoin(rmAppPageUrlBase, id.toString()))); return; + } else { + if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { + trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original); + } else { + trackingUri = new URI(original); + } } String runningUser = applicationReport.getUser(); @@ -311,8 +313,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) req.getQueryString(), true), runningUser, id); return; } - - URI toFetch = new URI(req.getScheme(), + URI toFetch = new URI(trackingUri.getScheme(), trackingUri.getAuthority(), StringHelper.ujoin(trackingUri.getPath(), rest), req.getQueryString(), null);