diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index 61e0429..26e8c42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -103,6 +103,12 @@ test + + org.bouncycastle + bcprov-jdk16 + test + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index f21ff2c..508433c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -47,6 +47,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -63,15 +64,16 @@ import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; +import org.apache.http.client.config.CookieSpecs; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.params.ClientPNames; -import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.conn.params.ConnRoutePNames; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.DefaultHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,6 +101,7 @@ private final String ahsAppPageUrlBase; private final String failurePageUrlBase; private transient YarnConfiguration conf; + private transient HttpClientBuilder httpClientBuilder; /** * HTTP methods. @@ -141,6 +144,16 @@ public WebAppProxyServlet() { StringHelper.pjoin(WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf), "applicationhistory", "app"); + httpClientBuilder = HttpClientBuilder.create(); + final SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + try { + sslFactory.init(); + httpClientBuilder.setSSLSocketFactory(new SSLConnectionSocketFactory( + sslFactory.createSSLSocketFactory(), sslFactory.getHostnameVerifier())); + } catch (Exception e) { + sslFactory.destroy(); + throw new RuntimeException(e); + } } /** @@ -186,17 +199,14 @@ private static void warnUserPage(HttpServletResponse resp, String link, * @param c the cookie to set if any * @param proxyHost the proxy host * @param method the http method + * @param httpClientBuilder the HttpClientBuilder * @throws IOException on any error. */ private static void proxyLink(final HttpServletRequest req, final HttpServletResponse resp, final URI link, final Cookie c, - final String proxyHost, final HTTP method) throws IOException { - DefaultHttpClient client = new DefaultHttpClient(); - client - .getParams() - .setParameter(ClientPNames.COOKIE_POLICY, - CookiePolicy.BROWSER_COMPATIBILITY) - .setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true); + final String proxyHost, final HTTP method, + HttpClientBuilder httpClientBuilder) throws IOException { + CloseableHttpClient client = httpClientBuilder.build(); // Make sure we send the request from the proxy address in the config // since that is what the AM filter checks against. IP aliasing or // similar could cause issues otherwise. @@ -204,8 +214,6 @@ private static void proxyLink(final HttpServletRequest req, if (LOG.isDebugEnabled()) { LOG.debug("local InetAddress for proxy host: {}", localAddress); } - client.getParams() - .setParameter(ConnRoutePNames.LOCAL_ADDRESS, localAddress); HttpRequestBase base = null; if (method.equals(HTTP.GET)) { @@ -228,6 +236,14 @@ private static void proxyLink(final HttpServletRequest req, return; } + RequestConfig requestConfig = RequestConfig.custom() + .setRedirectsEnabled(true) + .setCircularRedirectsAllowed(true) + .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY) + .setLocalAddress(localAddress) + .build(); + base.setConfig(requestConfig); + @SuppressWarnings("unchecked") Enumeration names = req.getHeaderNames(); while (names.hasMoreElements()) { @@ -453,7 +469,7 @@ private void methodAction(final HttpServletRequest req, if (userWasWarned && userApproved) { c = makeCheckCookie(id, true); } - proxyLink(req, resp, toFetch, c, getProxyHost(), method); + proxyLink(req, resp, toFetch, c, getProxyHost(), method, httpClientBuilder); } catch(URISyntaxException | YarnException e) { throw new IOException(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index fc97387..4b84f12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -30,11 +30,8 @@ import java.net.ConnectException; import java.net.HttpCookie; import java.net.HttpURLConnection; -import java.net.URI; import java.net.URL; import java.util.Enumeration; -import java.util.List; -import java.util.Map; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -42,18 +39,8 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -145,12 +132,12 @@ public void testWebAppProxyServlet() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS configuration.setInt("hadoop.http.max.threads", 10); - WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(); + WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(originalPort); proxy.init(configuration); proxy.start(); - int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort(); - AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher; + final int proxyPort = proxy.getProxyPort(); + WebAppProxyServerForTest.AppReportFetcherForTest appReportFetcher = proxy.getAppReportFetcher(); // wrong url try { @@ -177,7 +164,7 @@ public void testWebAppProxyServlet() throws Exception { proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true"); proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode()); - assertTrue(isResponseCookiePresent( + assertTrue(WebAppProxyServerForTest.isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); // test that redirection is squashed correctly @@ -199,27 +186,27 @@ public void testWebAppProxyServlet() throws Exception { + "page of the RM", expected, redirect); // cannot found application 1: null - appReportFetcher.answer = 1; + appReportFetcher.setAnswer(1); proxyConn = (HttpURLConnection) url.openConnection(); proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true"); proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_NOT_FOUND, proxyConn.getResponseCode()); - assertFalse(isResponseCookiePresent( + assertFalse(WebAppProxyServerForTest.isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); // cannot found application 2: ApplicationNotFoundException - appReportFetcher.answer = 4; + appReportFetcher.setAnswer(4); proxyConn = (HttpURLConnection) url.openConnection(); proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true"); proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_NOT_FOUND, proxyConn.getResponseCode()); - assertFalse(isResponseCookiePresent( + assertFalse(WebAppProxyServerForTest.isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); // wrong user - appReportFetcher.answer = 2; + appReportFetcher.setAnswer(2); proxyConn = (HttpURLConnection) url.openConnection(); proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode()); @@ -229,7 +216,7 @@ public void testWebAppProxyServlet() throws Exception { assertTrue(s.contains("WARNING: The following page may not be safe!")); //case if task has a not running status - appReportFetcher.answer = 3; + appReportFetcher.setAnswer(3); proxyConn = (HttpURLConnection) url.openConnection(); proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true"); proxyConn.connect(); @@ -237,7 +224,7 @@ public void testWebAppProxyServlet() throws Exception { // test user-provided path and query parameter can be appended to the // original tracking url - appReportFetcher.answer = 5; + appReportFetcher.setAnswer(5); URL clientUrl = new URL("http://localhost:" + proxyPort + "/proxy/application_00_0/test/tez?x=y&h=p"); proxyConn = (HttpURLConnection) clientUrl.openConnection(); @@ -256,19 +243,19 @@ public void testAppReportForEmptyTrackingUrl() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS configuration.setInt("hadoop.http.max.threads", 10); - WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(); + WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(originalPort); proxy.init(configuration); proxy.start(); - int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort(); - AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher; + final int proxyPort = proxy.getProxyPort(); + WebAppProxyServerForTest.AppReportFetcherForTest appReportFetcher = proxy.getAppReportFetcher(); try { //set AHS_ENBALED = false to simulate getting the app report from RM configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, false); ApplicationId app = ApplicationId.newInstance(0, 0); - appReportFetcher.answer = 6; + appReportFetcher.setAnswer(6); URL url = new URL("http://localhost:" + proxyPort + "/proxy/" + app.toString()); HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection(); @@ -314,11 +301,12 @@ public void testWebAppProxyPassThroughHeaders() throws Exception { Configuration configuration = new Configuration(); configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9091"); configuration.setInt("hadoop.http.max.threads", 10); - WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(); + WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(originalPort); proxy.init(configuration); proxy.start(); - int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort(); + final int proxyPort = proxy.getProxyPort(); + WebAppProxyServerForTest.AppReportFetcherForTest appReportFetcher = proxy.getAppReportFetcher(); try { URL url = new URL("http://localhost:" + proxyPort + "/proxy/application_00_1"); @@ -333,12 +321,12 @@ public void testWebAppProxyPassThroughHeaders() throws Exception { assertEquals(proxyConn.getRequestProperties().size(), 4); proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode()); - // Verify if number of headers received by end server is 8. - // Eight headers include Accept, Host, Connection, User-Agent, Cookie, - // Origin, Access-Control-Request-Method and + // Verify if number of headers received by end server is 9. + // Nine headers include Accept, Host, Connection, User-Agent, Cookie, + // Origin, Access-Control-Request-Method, Accept-Encoding and // Access-Control-Request-Headers. Pls note that Unknown-Header is dropped // by proxy as it is not in the list of allowed headers. - assertEquals(numberOfHeaders, 8); + assertEquals(numberOfHeaders, 9); assertFalse(hasUnknownHeader); } finally { proxy.close(); @@ -393,22 +381,6 @@ private String readInputStream(InputStream input) throws Exception { return new String(data.toByteArray(), "UTF-8"); } - private boolean isResponseCookiePresent(HttpURLConnection proxyConn, - String expectedName, String expectedValue) { - Map> headerFields = proxyConn.getHeaderFields(); - List cookiesHeader = headerFields.get("Set-Cookie"); - if (cookiesHeader != null) { - for (String cookie : cookiesHeader) { - HttpCookie c = HttpCookie.parse(cookie).get(0); - if (c.getName().equals(expectedName) - && c.getValue().equals(expectedValue)) { - return true; - } - } - } - return false; - } - @AfterClass public static void stop() throws Exception { try { @@ -422,129 +394,4 @@ public static void stop() throws Exception { } } - private class WebAppProxyServerForTest extends CompositeService { - - private WebAppProxyForTest proxy = null; - - public WebAppProxyServerForTest() { - super(WebAppProxyServer.class.getName()); - } - - @Override - public synchronized void serviceInit(Configuration conf) throws Exception { - proxy = new WebAppProxyForTest(); - addService(proxy); - super.serviceInit(conf); - } - - } - - private class WebAppProxyForTest extends WebAppProxy { - - HttpServer2 proxyServer; - AppReportFetcherForTest appReportFetcher; - - @Override - protected void serviceStart() throws Exception { - Configuration conf = getConfig(); - String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); - bindAddress = StringUtils.split(bindAddress, ':')[0]; - AccessControlList acl = new AccessControlList( - conf.get(YarnConfiguration.YARN_ADMIN_ACL, - YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); - proxyServer = new HttpServer2.Builder() - .setName("proxy") - .addEndpoint( - URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress - + ":0")).setFindPort(true) - .setConf(conf) - .setACL(acl) - .build(); - proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME, - ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); - - appReportFetcher = new AppReportFetcherForTest(conf); - proxyServer.setAttribute(FETCHER_ATTRIBUTE, - appReportFetcher ); - proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); - - String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] proxyParts = proxy.split(":"); - String proxyHost = proxyParts[0]; - - proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); - proxyServer.start(); - LOG.info("Proxy server is started at port {}", - proxyServer.getConnectorAddress(0).getPort()); - } - - } - - private class AppReportFetcherForTest extends AppReportFetcher { - int answer = 0; - - public AppReportFetcherForTest(Configuration conf) { - super(conf); - } - - public FetchedAppReport getApplicationReport(ApplicationId appId) - throws YarnException { - if (answer == 0) { - return getDefaultApplicationReport(appId); - } else if (answer == 1) { - return null; - } else if (answer == 2) { - FetchedAppReport result = getDefaultApplicationReport(appId); - result.getApplicationReport().setUser("user"); - return result; - } else if (answer == 3) { - FetchedAppReport result = getDefaultApplicationReport(appId); - result.getApplicationReport(). - setYarnApplicationState(YarnApplicationState.KILLED); - return result; - } else if (answer == 4) { - throw new ApplicationNotFoundException("Application is not found"); - } else if (answer == 5) { - // test user-provided path and query parameter can be appended to the - // original tracking url - FetchedAppReport result = getDefaultApplicationReport(appId); - result.getApplicationReport().setOriginalTrackingUrl("localhost:" - + originalPort + "/foo/bar?a=b#main"); - result.getApplicationReport(). - setYarnApplicationState(YarnApplicationState.FINISHED); - return result; - } else if (answer == 6) { - return getDefaultApplicationReport(appId, false); - } - return null; - } - - /* - * If this method is called with isTrackingUrl=false, no tracking url - * will set in the app report. Hence, there will be a connection exception - * when the prxyCon tries to connect. - */ - private FetchedAppReport getDefaultApplicationReport(ApplicationId appId, - boolean isTrackingUrl) { - FetchedAppReport fetchedReport; - ApplicationReport result = new ApplicationReportPBImpl(); - result.setApplicationId(appId); - result.setYarnApplicationState(YarnApplicationState.RUNNING); - result.setUser(CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER); - if (isTrackingUrl) { - result.setOriginalTrackingUrl("localhost:" + originalPort + "/foo/bar"); - } - if(configuration.getBoolean(YarnConfiguration. - APPLICATION_HISTORY_ENABLED, false)) { - fetchedReport = new FetchedAppReport(result, AppReportSource.AHS); - } else { - fetchedReport = new FetchedAppReport(result, AppReportSource.RM); - } - return fetchedReport; - } - - private FetchedAppReport getDefaultApplicationReport(ApplicationId appId) { - return getDefaultApplicationReport(appId, true); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletWithSSL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletWithSSL.java new file mode 100644 index 0000000..828c795 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletWithSSL.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.net.HttpCookie; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test the WebAppProxyServlet and WebAppProxy. For back end use simple web + * server. + */ +public class TestWebAppProxyServletWithSSL { + private static final String BASEDIR = System.getProperty("test.build.dir", + "target/test-dir") + "/" + TestWebAppProxyServletWithSSL.class.getSimpleName(); + private static final Logger LOG = LoggerFactory.getLogger( + TestWebAppProxyServletWithSSL.class); + + private static String keystoresDir; + private static HttpServer2 server; + private static String sslConfDir; + private static int originalPort = 0; + private static final Configuration configuration = new Configuration(); + + /** + * Simple https server. Server should send answer with status 200 + */ + @BeforeClass + public static void start() throws Exception { + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebAppProxyServletWithSSL.class); + + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, configuration, false); + Configuration sslConf = new Configuration(false); + sslConf.addResource("ssl-server.xml"); + sslConf.addResource("ssl-client.xml"); + + server = new HttpServer2.Builder() + .setName("test") + .addEndpoint(new URI("https://localhost")) + .setConf(configuration) + .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) + .keyStore(sslConf.get("ssl.server.keystore.location"), + sslConf.get("ssl.server.keystore.password"), + sslConf.get("ssl.server.keystore.type", "jks")) + .trustStore(sslConf.get("ssl.server.truststore.location"), + sslConf.get("ssl.server.truststore.password"), + sslConf.get("ssl.server.truststore.type", "jks")).build(); + server.addServlet("foobar" , "/foo/bar", TestWebAppProxyServlet.TestServlet.class); + server.start(); + URL baseUrl = new URL("https://" + + NetUtils.getHostPortString(server.getConnectorAddress(0))); + originalPort = baseUrl.getPort(); + LOG.info("Running embedded HTTPS servlet container at: " + baseUrl); + } + + @Test(timeout=500000) + public void testWebAppProxyServlet() throws Exception { + + configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); + // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS + configuration.setInt("hadoop.http.max.threads", 5); + WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(originalPort, "https"); + proxy.init(configuration); + proxy.start(); + + int proxyPort = proxy.getProxyPort(); + + try { + // set true Application ID in url + URL url = new URL("http://localhost:" + proxyPort + "/proxy/application_00_0"); + HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection(); + // set cookie + proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true"); + proxyConn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode()); + assertTrue(WebAppProxyServerForTest.isResponseCookiePresent( + proxyConn, "checked_application_0_0000", "true")); + } finally { + proxy.close(); + } + } + + @AfterClass + public static void stop() { + try { + server.stop(); + } catch (Exception e) { + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServerForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServerForTest.java new file mode 100644 index 0000000..302c9e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServerForTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import java.net.HttpCookie; +import java.net.HttpURLConnection; +import java.net.URI; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class WebAppProxyServerForTest extends CompositeService { + + private static final Logger LOG = LoggerFactory.getLogger(WebAppProxyServerForTest.class); + + private WebAppProxyForTest proxy = null; + private final String trackingUrlProtocol; + private final int originalPort; + + public WebAppProxyServerForTest(int originalPort, String trackingUrlProtocol) { + super(WebAppProxyServer.class.getName()); + this.originalPort = originalPort; + if (trackingUrlProtocol == null || trackingUrlProtocol.isEmpty()) { + this.trackingUrlProtocol = ""; + } else { + this.trackingUrlProtocol = trackingUrlProtocol + "://"; + } + } + + public WebAppProxyServerForTest(int originalPort) { + this(originalPort, ""); + } + + public static boolean isResponseCookiePresent(HttpURLConnection proxyConn, + String expectedName, String expectedValue) { + Map> headerFields = proxyConn.getHeaderFields(); + List cookiesHeader = headerFields.get("Set-Cookie"); + if (cookiesHeader != null) { + for (String cookie : cookiesHeader) { + HttpCookie c = HttpCookie.parse(cookie).get(0); + if (c.getName().equals(expectedName) + && c.getValue().equals(expectedValue)) { + return true; + } + } + } + return false; + } + + @Override + public synchronized void serviceInit(Configuration conf) throws Exception { + proxy = new WebAppProxyForTest(); + addService(proxy); + super.serviceInit(conf); + } + + public int getProxyPort() { + return proxy.proxyServer.getConnectorAddress(0).getPort(); + } + + public AppReportFetcherForTest getAppReportFetcher() { + return proxy.appReportFetcher; + } + + private class WebAppProxyForTest extends WebAppProxy { + + HttpServer2 proxyServer; + AppReportFetcherForTest appReportFetcher; + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); + bindAddress = StringUtils.split(bindAddress, ':')[0]; + AccessControlList acl = new AccessControlList( + conf.get(YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + proxyServer = new HttpServer2.Builder() + .setName("proxy") + .addEndpoint( + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + + ":0")).setFindPort(true) + .setConf(conf) + .setACL(acl) + .build(); + proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME, + ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); + + appReportFetcher = new AppReportFetcherForTest(conf); + proxyServer.setAttribute(FETCHER_ATTRIBUTE, + appReportFetcher ); + proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); + + String proxy = WebAppUtils.getProxyHostAndPort(conf); + String[] proxyParts = proxy.split(":"); + String proxyHost = proxyParts[0]; + + proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); + proxyServer.start(); + + LOG.info("Proxy server is started at port {}", + proxyServer.getConnectorAddress(0).getPort()); + } + + } + + public class AppReportFetcherForTest extends AppReportFetcher { + + private int answer = 0; + + public AppReportFetcherForTest(Configuration conf) { + super(conf); + } + + public void setAnswer(int answer) { + this.answer = answer; + } + + public FetchedAppReport getApplicationReport(ApplicationId appId) + throws YarnException { + if (answer == 0) { + return getDefaultApplicationReport(appId); + } else if (answer == 1) { + return null; + } else if (answer == 2) { + FetchedAppReport result = getDefaultApplicationReport(appId); + result.getApplicationReport().setUser("user"); + return result; + } else if (answer == 3) { + FetchedAppReport result = getDefaultApplicationReport(appId); + result.getApplicationReport(). + setYarnApplicationState(YarnApplicationState.KILLED); + return result; + } else if (answer == 4) { + throw new ApplicationNotFoundException("Application is not found"); + } else if (answer == 5) { + // test user-provided path and query parameter can be appended to the + // original tracking url + FetchedAppReport result = getDefaultApplicationReport(appId); + result.getApplicationReport().setOriginalTrackingUrl("localhost:" + + originalPort + "/foo/bar?a=b#main"); + result.getApplicationReport(). + setYarnApplicationState(YarnApplicationState.FINISHED); + return result; + } else if (answer == 6) { + return getDefaultApplicationReport(appId, false); + } + return null; + } + + /* + * If this method is called with isTrackingUrl=false, no tracking url + * will set in the app report. Hence, there will be a connection exception + * when the proxyCon tries to connect. + */ + private FetchedAppReport getDefaultApplicationReport(ApplicationId appId, + boolean isTrackingUrl) { + FetchedAppReport fetchedReport; + ApplicationReport result = new ApplicationReportPBImpl(); + result.setApplicationId(appId); + result.setYarnApplicationState(YarnApplicationState.RUNNING); + result.setUser(CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER); + if (isTrackingUrl) { + result.setOriginalTrackingUrl(trackingUrlProtocol + "localhost:" + originalPort + "/foo/bar"); + } + if(getConfig().getBoolean(YarnConfiguration. + APPLICATION_HISTORY_ENABLED, false)) { + fetchedReport = new FetchedAppReport(result, AppReportSource.AHS); + } else { + fetchedReport = new FetchedAppReport(result, AppReportSource.RM); + } + return fetchedReport; + } + + private FetchedAppReport getDefaultApplicationReport(ApplicationId appId) { + return getDefaultApplicationReport(appId, true); + } + } + +}