diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java new file mode 100644 index 0000000..dab9f28 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.util.Collection; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.ha.HAServiceTarget; +import org.apache.hadoop.yarn.client.RMHAServiceTarget; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +@Private +@Unstable +public class RMHAUtils { + + public static String findActiveRMHAId(YarnConfiguration conf) { + YarnConfiguration yarnConf = new YarnConfiguration(conf); + Collection rmIds = + yarnConf.getStringCollection(YarnConfiguration.RM_HA_IDS); + for (String currentId : rmIds) { + yarnConf.set(YarnConfiguration.RM_HA_ID, currentId); + try { + HAServiceState haState = getHAState(yarnConf); + if (haState.equals(HAServiceState.ACTIVE)) { + return currentId; + } + } catch (Exception e) { + // Couldn't check if this RM is active. Do nothing. Worst case, + // we wouldn't find an Active RM and return null. + } + } + return null; // Couldn't find an Active RM + } + + private static HAServiceState getHAState(YarnConfiguration yarnConf) + throws Exception { + HAServiceTarget haServiceTarget; + int rpcTimeoutForChecks = + yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, + CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); + + yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, + yarnConf.get(YarnConfiguration.RM_PRINCIPAL, "")); + haServiceTarget = new RMHAServiceTarget(yarnConf); + HAServiceProtocol proto = + haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); + HAServiceState haState = proto.getServiceStatus().getState(); + return haState; + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 61ac708..f3c4c3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -22,6 +22,9 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -81,6 +84,33 @@ public static String getRMWebAppURLWithoutScheme(Configuration conf) { YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); } } + + public static List getProxyHostsAndPortsForAmFilter( + Configuration conf) { + List addrs = new ArrayList(); + String proxyAddr = conf.get(YarnConfiguration.PROXY_ADDRESS); + if (proxyAddr == null || proxyAddr.isEmpty()) { + Map addrsMap = YarnConfiguration.useHttps(conf) + ? conf.getValByRegex(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS) + : conf.getValByRegex(YarnConfiguration.RM_WEBAPP_ADDRESS); + if (!addrsMap.isEmpty()) { + for (String addr : addrsMap.values()) { + try { + InetSocketAddress socketAddr = NetUtils.createSocketAddr(addr); + addrs.add(getResolvedAddress(socketAddr)); + } catch(IllegalArgumentException e) { + // skip if can't resolve + } + } + } + if (addrs.isEmpty()) { + addrs.add(getResolvedRMWebAppURLWithoutScheme(conf)); + } + } else { + addrs.add(proxyAddr); + } + return addrs; + } public static String getProxyHostAndPort(Configuration conf) { String addr = conf.get(YarnConfiguration.PROXY_ADDRESS); @@ -114,10 +144,14 @@ public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); } + return getResolvedAddress(address); + } + + private static String getResolvedAddress(InetSocketAddress address) { address = NetUtils.getConnectAddress(address); - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); InetAddress resolved = address.getAddress(); - if (resolved == null || resolved.isAnyLocalAddress() || + if (resolved == null || resolved.isAnyLocalAddress() || resolved.isLoopbackAddress()) { String lh = address.getHostName(); try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java new file mode 100644 index 0000000..236bfcd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java @@ -0,0 +1,96 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.webapp.util; + +import java.util.Collections; +import java.util.List; +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +public class TestWebAppUtils extends TestCase { + + @Test + public void testGetProxyHostsAndPortsForAmFilter() { + NetUtils.addStaticResolution("host1", "172.0.0.1"); + NetUtils.addStaticResolution("host2", "172.0.0.1"); + NetUtils.addStaticResolution("host3", "172.0.0.1"); + NetUtils.addStaticResolution("host4", "172.0.0.1"); + NetUtils.addStaticResolution("host5", "172.0.0.1"); + NetUtils.addStaticResolution("host6", "172.0.0.1"); + + // Check no configs given + Configuration conf = new Configuration(false); + List proxyHosts = + WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(1, proxyHosts.size()); + assertEquals(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf), + proxyHosts.get(0)); + + // Check PROXY_ADDRESS has priority + conf = new Configuration(false); + conf.set(YarnConfiguration.PROXY_ADDRESS, "host1:1000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm1", "host2:2000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm2", "host3:3000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm3", "host4:4000"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(1, proxyHosts.size()); + assertEquals("host1:1000", proxyHosts.get(0)); + + // Check getting a single RM_WEBAPP_ADDRESS + conf = new Configuration(false); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "host2:2000"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(1, proxyHosts.size()); + Collections.sort(proxyHosts); + assertEquals("host2:2000", proxyHosts.get(0)); + + // Check getting multiple RM_WEBAPP_ADDRESSes (RM HA) + conf = new Configuration(false); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm1", "host2:2000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm2", "host3:3000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm3", "host4:4000"); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm1", "host5:5000"); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm2", "host6:6000"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(3, proxyHosts.size()); + Collections.sort(proxyHosts); + assertEquals("host2:2000", proxyHosts.get(0)); + assertEquals("host3:3000", proxyHosts.get(1)); + assertEquals("host4:4000", proxyHosts.get(2)); + + // Check getting multiple RM_WEBAPP_ADDRESSes (RM HA) with HTTPS + conf = new Configuration(false); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTPS_ONLY.toString()); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm1", "host2:2000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm2", "host3:3000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm3", "host4:4000"); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm1", "host5:5000"); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm2", "host6:6000"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(2, proxyHosts.size()); + Collections.sort(proxyHosts); + assertEquals("host5:5000", proxyHosts.get(0)); + assertEquals("host6:6000", proxyHosts.get(1)); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java deleted file mode 100644 index a964875..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager; - -import java.util.Collection; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.ha.HAServiceProtocol; -import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.ha.HAServiceTarget; -import org.apache.hadoop.yarn.client.RMHAServiceTarget; -import org.apache.hadoop.yarn.conf.YarnConfiguration; - -@Private -@Unstable -public class RMHAUtils { - - public static String findActiveRMHAId(YarnConfiguration conf) { - YarnConfiguration yarnConf = new YarnConfiguration(conf); - Collection rmIds = - yarnConf.getStringCollection(YarnConfiguration.RM_HA_IDS); - for (String currentId : rmIds) { - yarnConf.set(YarnConfiguration.RM_HA_ID, currentId); - try { - HAServiceState haState = getHAState(yarnConf); - if (haState.equals(HAServiceState.ACTIVE)) { - return currentId; - } - } catch (Exception e) { - // Couldn't check if this RM is active. Do nothing. Worst case, - // we wouldn't find an Active RM and return null. - } - } - return null; // Couldn't find an Active RM - } - - private static HAServiceState getHAState(YarnConfiguration yarnConf) - throws Exception { - HAServiceTarget haServiceTarget; - int rpcTimeoutForChecks = - yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, - CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); - - yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, - yarnConf.get(YarnConfiguration.RM_PRINCIPAL, "")); - haServiceTarget = new RMHAServiceTarget(yarnConf); - HAServiceProtocol proto = - haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); - HAServiceState haState = proto.getServiceStatus().getState(); - return haState; - } -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index fe4a592..ffb693e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -25,7 +25,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.RMHAUtils; +import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index 6ed56f7..fcc9b95 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -36,11 +37,23 @@ @Override public void initFilter(FilterContainer container, Configuration conf) { Map params = new HashMap(); - String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] parts = proxy.split(":"); - params.put(AmIpFilter.PROXY_HOST, parts[0]); - params.put(AmIpFilter.PROXY_URI_BASE, WebAppUtils.getHttpSchemePrefix(conf) - + proxy + getApplicationWebProxyBase()); + List proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + StringBuilder sb = new StringBuilder(); + for (String proxy : proxies) { + sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER); + } + sb.setLength(sb.length() - 1); + params.put(AmIpFilter.PROXY_HOSTS, sb.toString()); + + String prefix = WebAppUtils.getHttpSchemePrefix(conf); + String proxyBase = getApplicationWebProxyBase(); + sb = new StringBuilder(); + for (String proxy : proxies) { + sb.append(prefix).append(proxy).append(proxyBase) + .append(AmIpFilter.PROXY_HOSTS_DELIMITER); + } + sb.setLength(sb.length() - 1); + params.put(AmIpFilter.PROXY_URI_BASES, sb.toString()); container.addFilter(FILTER_NAME, FILTER_CLASS, params); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index fbf87bd..46cf8bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; import java.net.UnknownHostException; import java.util.HashSet; import java.util.Set; @@ -36,42 +38,53 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.apache.hadoop.yarn.util.RMHAUtils; public class AmIpFilter implements Filter { private static final Log LOG = LogFactory.getLog(AmIpFilter.class); - - public static final String PROXY_HOST = "PROXY_HOST"; - public static final String PROXY_URI_BASE = "PROXY_URI_BASE"; + + public static final String PROXY_HOSTS = "PROXY_HOSTS"; + public static final String PROXY_HOSTS_DELIMITER = ","; + public static final String PROXY_URI_BASES = "PROXY_URI_BASES"; + public static final String PROXY_URI_BASES_DELIMITER = ","; //update the proxy IP list about every 5 min private static final long updateInterval = 5 * 60 * 1000; - - private String proxyHost; + + private String[] proxyHosts; private Set proxyAddresses = null; private long lastUpdate; - private String proxyUriBase; - + private String[] proxyUriBases; + @Override public void init(FilterConfig conf) throws ServletException { - proxyHost = conf.getInitParameter(PROXY_HOST); - proxyUriBase = conf.getInitParameter(PROXY_URI_BASE); + proxyHosts = conf.getInitParameter(PROXY_HOSTS) + .split(PROXY_HOSTS_DELIMITER); + proxyUriBases = conf.getInitParameter(PROXY_URI_BASES) + .split(PROXY_URI_BASES_DELIMITER); } - + protected Set getProxyAddresses() throws ServletException { long now = System.currentTimeMillis(); synchronized(this) { if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) { - try { - proxyAddresses = new HashSet(); - for(InetAddress add : InetAddress.getAllByName(proxyHost)) { - if (LOG.isDebugEnabled()) { - LOG.debug("proxy address is: " + add.getHostAddress()); + proxyAddresses = new HashSet(); + for (String proxyHost : proxyHosts) { + try { + for(InetAddress add : InetAddress.getAllByName(proxyHost)) { + if (LOG.isDebugEnabled()) { + LOG.debug("proxy address is: " + add.getHostAddress()); + } + proxyAddresses.add(add.getHostAddress()); + } + lastUpdate = now; + } catch (UnknownHostException e) { + LOG.warn("Could not locate " + proxyHost + " - skipping", e); } - proxyAddresses.add(add.getHostAddress()); } - lastUpdate = now; - } catch (UnknownHostException e) { - throw new ServletException("Could not locate "+proxyHost, e); + if (proxyAddresses.isEmpty()) { + throw new ServletException("Could not locate any of the proxy hosts"); } } return proxyAddresses; @@ -89,21 +102,22 @@ public void doFilter(ServletRequest req, ServletResponse resp, if(!(req instanceof HttpServletRequest)) { throw new ServletException("This filter only works for HTTP/HTTPS"); } - + HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletResponse httpResp = (HttpServletResponse)resp; if (LOG.isDebugEnabled()) { LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr()); } if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) { - String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + + String redirectUrl = findRedirectUrl(); + redirectUrl = httpResp.encodeRedirectURL(redirectUrl + httpReq.getRequestURI()); httpResp.sendRedirect(redirectUrl); return; } - + String user = null; - + if (httpReq.getCookies() != null) { for(Cookie c: httpReq.getCookies()) { if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){ @@ -118,9 +132,39 @@ public void doFilter(ServletRequest req, ServletResponse resp, chain.doFilter(req, resp); } else { final AmIpPrincipal principal = new AmIpPrincipal(user); - ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, + ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, principal); chain.doFilter(requestWrapper, resp); } } + + protected String findRedirectUrl() throws ServletException { + String addr = null; + if (proxyUriBases.length == 1) { // external proxy or not HA + addr = proxyUriBases[0]; + } else { + YarnConfiguration conf = new YarnConfiguration(); + String activeRMId = RMHAUtils.findActiveRMHAId(conf); + String host = YarnConfiguration.useHttps(conf) + ? conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + + "." + activeRMId) + : conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + activeRMId); + for (String proxyUriBase : proxyUriBases) { + try { + URL url = new URL(proxyUriBase); + if (host.equals(url.getHost() + ":" + url.getPort())) { + addr = proxyUriBase; + break; + } + } catch(MalformedURLException e) { + // ignore + } + } + } + if (addr == null) { + throw new ServletException( + "Could not determine the proxy server for redirection"); + } + return addr; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java index e75cca5..16f0084 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java @@ -106,8 +106,8 @@ public void doFilter(ServletRequest servletRequest, }; Map params = new HashMap(); - params.put(AmIpFilter.PROXY_HOST, proxyHost); - params.put(AmIpFilter.PROXY_URI_BASE, proxyUri); + params.put(AmIpFilter.PROXY_HOSTS, proxyHost); + params.put(AmIpFilter.PROXY_URI_BASES, proxyUri); FilterConfig conf = new DummyFilterConfig(params); Filter filter = new TestAmIpFilter(); filter.init(conf); @@ -122,8 +122,8 @@ public void doFilter(ServletRequest servletRequest, @Test(timeout = 1000) public void testFilter() throws Exception { Map params = new HashMap(); - params.put(AmIpFilter.PROXY_HOST, proxyHost); - params.put(AmIpFilter.PROXY_URI_BASE, proxyUri); + params.put(AmIpFilter.PROXY_HOSTS, proxyHost); + params.put(AmIpFilter.PROXY_URI_BASES, proxyUri); FilterConfig config = new DummyFilterConfig(params); // dummy filter diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java new file mode 100644 index 0000000..7b932a5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webproxy.amfilter; + +import java.util.Arrays; +import java.util.Map; +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +public class TestAmFilterInitializer extends TestCase { + + @Test + public void testInitFilter() { + NetUtils.addStaticResolution("host1", "172.0.0.1"); + NetUtils.addStaticResolution("host2", "172.0.0.1"); + NetUtils.addStaticResolution("host3", "172.0.0.1"); + NetUtils.addStaticResolution("host4", "172.0.0.1"); + NetUtils.addStaticResolution("host5", "172.0.0.1"); + NetUtils.addStaticResolution("host6", "172.0.0.1"); + + // Check PROXY_ADDRESS + MockFilterContainer con = new MockFilterContainer(); + Configuration conf = new Configuration(false); + conf.set(YarnConfiguration.PROXY_ADDRESS, "host1:1000"); + AmFilterInitializer afi = new MockAmFilterInitializer(); + assertNull(con.givenParameters); + afi.initFilter(con, conf); + assertEquals(2, con.givenParameters.size()); + assertEquals("host1", con.givenParameters.get(AmIpFilter.PROXY_HOSTS)); + assertEquals("http://host1:1000/foo", + con.givenParameters.get(AmIpFilter.PROXY_URI_BASES)); + + // Check a single RM_WEBAPP_ADDRESS + con = new MockFilterContainer(); + conf = new Configuration(false); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "host2:2000"); + afi = new MockAmFilterInitializer(); + assertNull(con.givenParameters); + afi.initFilter(con, conf); + assertEquals(2, con.givenParameters.size()); + assertEquals("host2", con.givenParameters.get(AmIpFilter.PROXY_HOSTS)); + assertEquals("http://host2:2000/foo", + con.givenParameters.get(AmIpFilter.PROXY_URI_BASES)); + + // Check multiple RM_WEBAPP_ADDRESSes (RM HA) + con = new MockFilterContainer(); + conf = new Configuration(false); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm1", "host2:2000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm2", "host3:3000"); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + ".rm3", "host4:4000"); + afi = new MockAmFilterInitializer(); + assertNull(con.givenParameters); + afi.initFilter(con, conf); + assertEquals(2, con.givenParameters.size()); + String[] proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS) + .split(AmIpFilter.PROXY_HOSTS_DELIMITER); + assertEquals(3, proxyHosts.length); + Arrays.sort(proxyHosts); + assertEquals("host2", proxyHosts[0]); + assertEquals("host3", proxyHosts[1]); + assertEquals("host4", proxyHosts[2]); + String[] proxyBases = con.givenParameters.get(AmIpFilter.PROXY_URI_BASES) + .split(AmIpFilter.PROXY_URI_BASES_DELIMITER); + assertEquals(3, proxyBases.length); + Arrays.sort(proxyBases); + assertEquals("http://host2:2000/foo", proxyBases[0]); + assertEquals("http://host3:3000/foo", proxyBases[1]); + assertEquals("http://host4:4000/foo", proxyBases[2]); + + // Check multiple RM_WEBAPP_ADDRESSes (RM HA) with HTTPS + con = new MockFilterContainer(); + conf = new Configuration(false); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTPS_ONLY.toString()); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm1", "host5:5000"); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm2", "host6:6000"); + afi = new MockAmFilterInitializer(); + assertNull(con.givenParameters); + afi.initFilter(con, conf); + assertEquals(2, con.givenParameters.size()); + proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS) + .split(AmIpFilter.PROXY_HOSTS_DELIMITER); + assertEquals(2, proxyHosts.length); + Arrays.sort(proxyHosts); + assertEquals("host5", proxyHosts[0]); + assertEquals("host6", proxyHosts[1]); + proxyBases = con.givenParameters.get(AmIpFilter.PROXY_URI_BASES) + .split(AmIpFilter.PROXY_URI_BASES_DELIMITER); + assertEquals(2, proxyBases.length); + Arrays.sort(proxyBases); + assertEquals("https://host5:5000/foo", proxyBases[0]); + assertEquals("https://host6:6000/foo", proxyBases[1]); + } + + class MockAmFilterInitializer extends AmFilterInitializer { + @Override + protected String getApplicationWebProxyBase() { + return "/foo"; + } + } + + class MockFilterContainer implements FilterContainer { + Map givenParameters; + + @Override + public void addFilter(String name, String classname, Map parameters) { + givenParameters = parameters; + } + + @Override + public void addGlobalFilter(String name, String classname, + Map parameters) { + } + + } +}