diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 44ed223..92337a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -79,9 +79,22 @@ protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, protected void render(Block html) { String webUiType = $(WEB_UI_TYPE); String aid = $(APPLICATION_ID); + if (aid.isEmpty()) { puts("Bad request: requires Application ID"); return; + } else if (aid.endsWith("R")) { + aid = aid.substring(0, aid.length() - 1); + html.p()._("The application master for " + aid + " redirected the " + + "resource manager's web proxy's request back to the web proxy, " + + "which means your request to view the application master's web UI " + + "cannot be fulfilled. The typical cause for this error is a " + + "network misconfiguration that causes the resource manager's web " + + "proxy host to resolve to an unexpected IP address on the " + + "application master host. Please contact your cluster " + + "administrator to resolve the issue.")._(); + + return; } try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java index 0c5516a..4c0d77f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java @@ -33,10 +33,15 @@ protected void preHead(Page.HTML<_> html) { commonPreHead(html); String appId = $(YarnWebParams.APPLICATION_ID); + + if (appId.endsWith("R")) { + appId = appId.substring(0, appId.length() - 1); + } + set( - TITLE, - appId.isEmpty() ? "Bad request: missing application ID" : join( - "Application ", $(YarnWebParams.APPLICATION_ID))); + TITLE, appId.isEmpty() ? + "Bad request: missing application ID" : + join("Application ", appId)); set(DATATABLES_ID, "attempts ResourceRequests"); set(initID(DATATABLES, "attempts"), WebPageUtils.attemptsTableInit()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 0e988b8..4545dce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.net.InetAddress; +import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; @@ -44,6 +45,7 @@ import javax.ws.rs.core.UriBuilder; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -316,26 +318,37 @@ private void methodAction(final HttpServletRequest req, final String pathInfo = req.getPathInfo(); String[] parts = null; + if (pathInfo != null) { parts = pathInfo.split("/", 3); } - if(parts == null || parts.length < 2) { + + if (parts == null || parts.length < 2) { LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo); notFound(resp, "Your path appears to be formatted incorrectly."); return; } + //parts[0] is empty because path info always starts with a / String appId = parts[1]; String rest = parts.length > 2 ? parts[2] : ""; ApplicationId id = Apps.toAppID(appId); - if(id == null) { + + if (id == null) { LOG.warn("{} attempting to access {} that is invalid", remoteUser, appId); notFound(resp, appId + " appears to be formatted incorrectly."); return; } - - if(securityEnabled) { + + // If this call is from an AM redirect, we need to be careful about how + // we handle it. If this method returns true, it means the method + // already redirected the response, so we can just return. + if (handleRedirect(rest, appId, req, resp)) { + return; + } + + if (securityEnabled) { String cookieName = getCheckCookieName(id); Cookie[] cookies = req.getCookies(); if (cookies != null) { @@ -456,6 +469,56 @@ private void methodAction(final HttpServletRequest req, } /** + * Check whether the request is a redirect from the AM and handle it + * appropriately. This check exists to prevent the AM from forwarding back to + * the web proxy, which would contact the AM again, which would forward + * again... If this method returns true, there was a redirect, and + * it was handled by redirecting the current request to an error page. + * + * @param path the part of the request path after the app id + * @param id the app id + * @param req the request object + * @param resp the response object + * @return whether there was a redirect + * @throws IOException if a redirect fails + */ + private boolean handleRedirect(String path, String id, HttpServletRequest req, + HttpServletResponse resp) throws IOException { + // If this isn't a redirect, we don't care. + boolean handle = path.endsWith("/redirect"); + + if (handle) { + // If this is a redirect, check if we're calling ourselves. + try { + handle = NetUtils.getLocalInetAddress(req.getRemoteHost()) != null; + } catch (SocketException ex) { + // This exception means we can't determine the calling host. Odds are + // that means it's not us. Let it go and hope it works out bettor next + // time. + handle = false; + } + } + + // If the proxy tries to call itself, it gets into an endless + // loop and consumes all available handler threads until the + // application completes. Redirect to the app page with a flag + // that tells it to print an appropriate error message. + if (handle) { + LOG.error("The AM's web app redirected the RM web proxy's request back " + + "to the web proxy. The typical cause is that the AM is resolving " + + "the RM's address as something other than what it expects. Check " + + "your network configuration and the value of the " + + "yarn.web-proxy.address property. Once the host resolution issue " + + "has been resolved, you will likely need to delete the " + + "misbehaving application, " + id); + ProxyUtils.sendRedirect(req, resp, + StringHelper.pjoin(rmAppPageUrlBase, id + "R")); + } + + return handle; + } + + /** * This method is used by Java object deserialization, to fill in the * transient {@link #trackingUriPlugins} field. * See {@link ObjectInputStream#defaultReadObject()} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index e7617f0..e7da3e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -136,7 +136,10 @@ public void doFilter(ServletRequest req, ServletResponse resp, } if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) { String redirectUrl = findRedirectUrl(); - String target = redirectUrl + httpReq.getRequestURI(); + + // Add /redirect to the end so that the RM web proxy knows that this + // request was a redirect. + String target = redirectUrl + httpReq.getRequestURI() + "/redirect"; ProxyUtils.sendRedirect(httpReq, httpResp, target); return; }