diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1314bf9e67b..1b1cc3d050c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2276,6 +2276,17 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_RM_APPLICATION_HTTPS_POLICY = "NONE"; + public static final String RM_PROXY_HTTPCLIENT_SOCKET_TIMEOUT = + RM_PREFIX + "proxy.httpclient.timeout"; + + public static final int DEFAULT_RM_PROXY_HTTPCLIENT_SOCKET_TIMEOUT = + 60 * 1000; + + public static final String RM_PROXY_HTTPCLIENT_CONNECTION_TIMEOUT = + RM_PREFIX + "proxy.httpclient.timeout"; + + public static final int DEFAULT_RM_PROXY_HTTPCLIENT_CONNECTION_TIMEOUT = + 60 * 1000; /** * Interval of time the linux container executor should try cleaning up * cgroups entry when cleaning up a container. This is required due to what diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index ae9a01f6738..4063679095d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -122,6 +122,9 @@ } } + protected void setConf(YarnConfiguration conf){ + this.conf = conf; + } /** * Default constructor */ @@ -230,6 +233,13 @@ private void proxyLink(final HttpServletRequest req, String httpsPolicy = conf.get(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, YarnConfiguration.DEFAULT_RM_APPLICATION_HTTPS_POLICY); + int socketTimeout = + conf.getInt(YarnConfiguration.RM_PROXY_HTTPCLIENT_SOCKET_TIMEOUT, + YarnConfiguration.DEFAULT_RM_PROXY_HTTPCLIENT_SOCKET_TIMEOUT); + int connectionTimeout = + conf.getInt(YarnConfiguration.RM_PROXY_HTTPCLIENT_CONNECTION_TIMEOUT, + YarnConfiguration.DEFAULT_RM_PROXY_HTTPCLIENT_CONNECTION_TIMEOUT); + if (httpsPolicy.equals("LENIENT") || httpsPolicy.equals("STRICT")) { ProxyCA proxyCA = getProxyCA(); // ProxyCA could be null when the Proxy is run outside the RM @@ -253,6 +263,8 @@ private void proxyLink(final HttpServletRequest req, RequestConfig.custom() .setCircularRedirectsAllowed(true) .setLocalAddress(localAddress) + .setSocketTimeout(socketTimeout) + .setConnectTimeout(connectionTimeout) .build()); HttpRequestBase base = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 6cc1d22d582..3c5f1a550c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -18,23 +18,31 @@ package org.apache.hadoop.yarn.server.webproxy; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; +import org.apache.hadoop.io.IOUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringWriter; import java.net.ConnectException; import java.net.HttpCookie; import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.Map; @@ -63,11 +71,16 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.junit.AfterClass; +import static org.junit.Assert.fail; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.rules.ExpectedException; import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,6 +111,7 @@ public static void start() throws Exception { context.setContextPath("/foo"); server.setHandler(context); context.addServlet(new ServletHolder(TestServlet.class), "/bar"); + context.addServlet(new ServletHolder(LoopTestServlet.class), "/loop"); ((ServerConnector)server.getConnectors()[0]).setHost("localhost"); server.start(); originalPort = ((ServerConnector)server.getConnectors()[0]).getLocalPort(); @@ -145,6 +159,87 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) } } + @SuppressWarnings("serial") + public static class LoopTestServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + int numHeaders = 0; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + InputStream is = req.getInputStream(); + OutputStream os = resp.getOutputStream(); + int c = is.read(); + while (c > -1) { + os.write(c); + c = is.read(); + } + is.close(); + os.close(); + resp.setStatus(HttpServletResponse.SC_OK); + } + } + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test(timeout=8000) + public void test() throws IOException { + expectedException.expect(SocketTimeoutException.class); + expectedException.expectMessage("Read timed out"); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRemoteUser()).thenReturn("dr.who"); + when(request.getPathInfo()).thenReturn("/application_00_0"); + when(request.getCookies()).thenReturn(null); + when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration()); + HttpServletResponse response = mock(HttpServletResponse.class); + OutputStream os = new ByteArrayOutputStream(); + try { + when(response.getOutputStream()).thenReturn(null); + } catch (IOException e) { + e.printStackTrace(); + } + WebAppProxyServlet servlet = new WebAppProxyServlet(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.RM_PROXY_HTTPCLIENT_SOCKET_TIMEOUT, 3 * 1000); + conf.setInt(YarnConfiguration.RM_PROXY_HTTPCLIENT_CONNECTION_TIMEOUT, 3 * 1000); + servlet.setConf(conf); + + ServletConfig config = mock(ServletConfig.class); + ServletContext context = mock(ServletContext.class); + when(config.getServletContext()).thenReturn(context); + AppReportFetcherForTest appReportFetcher = + new AppReportFetcherForTest(new YarnConfiguration()); + when(config.getServletContext() + .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)) + .thenReturn(appReportFetcher); + appReportFetcher.answer = 7; + try { + servlet.init(config); + } catch (ServletException e) { + LOG.error(e.getMessage()); + fail("failed to init servlet"); + } + try { + servlet.doGet(request, response); + } catch (ServletException e) + { + LOG.error(e.getMessage()); + fail("ServletException thrown during doGet."); + } + } + @Test(timeout=5000) public void testWebAppProxyServlet() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); @@ -391,9 +486,9 @@ public void testWebAppProxyServerMainMethod() throws Exception { @Test(timeout=5000) public void testCheckHttpsStrictAndNotProvided() throws Exception { - HttpServletResponse resp = Mockito.mock(HttpServletResponse.class); + HttpServletResponse resp = mock(HttpServletResponse.class); StringWriter sw = new StringWriter(); - Mockito.when(resp.getWriter()).thenReturn(new PrintWriter(sw)); + when(resp.getWriter()).thenReturn(new PrintWriter(sw)); YarnConfiguration conf = new YarnConfiguration(); final URI httpLink = new URI("http://foo.com"); final URI httpsLink = new URI("https://foo.com"); @@ -566,6 +661,11 @@ public FetchedAppReport getApplicationReport(ApplicationId appId) return result; } else if (answer == 6) { return getDefaultApplicationReport(appId, false); + } else if (answer == 7) { + FetchedAppReport result = getDefaultApplicationReport(appId); + result.getApplicationReport().setOriginalTrackingUrl("localhost:" + + originalPort + "/foo/loop?a=b#main"); + return result; } return null; }