diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d42562cf614..94fd4c8d75d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -357,6 +357,10 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_RM_WEBAPP_HTTPS_PORT; + public static final String RM_WEBAPP_PROXY_ENABLE = + RM_PREFIX + "webapp.proxy.enable"; + public static final boolean DEFAULT_RM_WEBAPP_PROXY_ENABLE = true; + /** * Enable YARN WebApp V2. */ @@ -4095,6 +4099,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + public static final String ROUTER_WEBAPP_PROXY_ENABLE = + ROUTER_WEBAPP_PREFIX + "proxy.enable"; + public static final boolean DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE = true; + /** * Connection and Read timeout from the Router to RM. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 83ae269a6b5..b8b00ede004 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics; import org.apache.hadoop.yarn.server.webproxy.DefaultAppReportFetcher; +import org.apache.hadoop.yarn.server.webproxy.FedAppReportFetcher; import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1391,7 +1392,11 @@ protected void startWepApp() { String proxyHostAndPort = rmContext.getProxyHostAndPort(conf); if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf). equals(proxyHostAndPort)) { - if (HAUtil.isHAEnabled(conf)) { + if (HAUtil.isFederationEnabled(conf) && conf + .getBoolean(YarnConfiguration.RM_WEBAPP_PROXY_ENABLE, + YarnConfiguration.DEFAULT_RM_WEBAPP_PROXY_ENABLE)) { + fetcher = new FedAppReportFetcher(conf); + } else if (HAUtil.isHAEnabled(conf)) { fetcher = new DefaultAppReportFetcher(conf); } else { fetcher = new DefaultAppReportFetcher(conf, getClientRMService()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index fc0b9eeaef3..5ee1acaa8e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -36,6 +37,10 @@ import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService; import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp; +import org.apache.hadoop.yarn.server.webproxy.FedAppReportFetcher; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; @@ -75,6 +80,7 @@ private WebApp webApp; @VisibleForTesting protected String webAppAddress; + private FedAppReportFetcher fetcher = null; /** * Priority of the Router shutdown hook. @@ -168,9 +174,31 @@ public void startWepApp() { Builder builder = WebApps.$for("cluster", null, null, "ws").with(conf).at(webAppAddress); + if(conf.getBoolean(YarnConfiguration.ROUTER_WEBAPP_PROXY_ENABLE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE)) { + fetcher = new FedAppReportFetcher(conf); + builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, + ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); + builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); + String proxyHostAndPort = getProxyHostAndPort(conf); + String[] proxyParts = proxyHostAndPort.split(":"); + builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); + } webApp = builder.start(new RouterWebApp(this)); } + public static String getProxyHostAndPort(Configuration conf) { + String addr = conf.get(YarnConfiguration.PROXY_ADDRESS); + if(addr == null || addr.isEmpty()) { + InetSocketAddress address = + conf.getSocketAddr(YarnConfiguration.ROUTER_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PORT); + addr = WebAppUtils.getResolvedAddress(address); + } + return addr; + } + public static void main(String[] argv) { Configuration conf = new YarnConfiguration(); Thread diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 955c48fd953..5dbe9e0b913 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -112,6 +114,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,6 +144,8 @@ // and remove the oldest used ones. private Map userPipelineMap; + protected URL redirectURL; + public RouterClientRMService() { super(RouterClientRMService.class.getName()); } @@ -158,6 +163,11 @@ protected void serviceStart() throws Exception { YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_ADDRESS, YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_PORT); + if (getConfig().getBoolean(YarnConfiguration.ROUTER_WEBAPP_PROXY_ENABLE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE)) { + redirectURL = getRedirectURL(); + } + int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); @@ -339,7 +349,24 @@ public GetClusterNodeLabelsResponse getClusterNodeLabels( public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnException, IOException { RequestInterceptorChainWrapper pipeline = getInterceptorChain(); - return pipeline.getRootInterceptor().getApplicationReport(request); + GetApplicationReportResponse response = pipeline.getRootInterceptor() + .getApplicationReport(request); + if (getConfig().getBoolean(YarnConfiguration.ROUTER_WEBAPP_PROXY_ENABLE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE)) { + // redirect url + URL url = new URL(response.getApplicationReport().getTrackingUrl()); + String redirectUrl = new URL(redirectURL.getProtocol(), + redirectURL.getHost(), redirectURL.getPort(), url.getFile()) + .toString(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "The tracking url of application {} is redirect from {} to {}", + response.getApplicationReport().getApplicationId(), url, + redirectUrl); + } + response.getApplicationReport().setTrackingUrl(redirectUrl); + } + return response; } @Override @@ -616,4 +643,27 @@ protected void finalize() { rootInterceptor.shutdown(); } } + + private URL getRedirectURL() throws Exception { + Configuration conf = getConfig(); + String webAppAddress = WebAppUtils.getWebAppBindURL(conf, + YarnConfiguration.ROUTER_BIND_HOST, + WebAppUtils.getRouterWebAppURLWithoutScheme(conf)); + + String[] hostPort = StringUtils.split(webAppAddress, ':'); + if (hostPort.length != 2) { + throw new YarnRuntimeException( + "Router can't get valid redirect proxy url"); + } + String host; + if (null == hostPort[0] || hostPort[0].equals("") || hostPort[0] + .equals("0.0.0.0")) { + host = InetAddress.getLocalHost().getCanonicalHostName(); + } else { + host = hostPort[0]; + } + return new URL( + YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http", host, + Integer.parseInt(hostPort[1]), ""); + } }