diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 258b991..d198d34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -51,6 +51,9 @@ @Stable public abstract class ApplicationReport { + public static final String APP_REPORT_SOURCE_RM = "RM"; + public static final String APP_REPORT_SOURCE_AHS = "AHS"; + @Private @Unstable public static ApplicationReport newInstance(ApplicationId applicationId, @@ -104,6 +107,22 @@ public static ApplicationReport newInstance(ApplicationId applicationId, } /** + * Get the appReportSource where the appReport was fetched from + * @return appReportSource where the appReport was fetched from + */ + @Public + @Stable + public abstract String getAppReportSource(); + + /** + * Set the appReportSource where the appReport was fetched from + * @param appReportSource + */ + @Private + @Unstable + public abstract void setAppReportSource(String appReportString); + + /** * Get the ApplicationId of the application. * @return ApplicationId of the application */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 2e50e0d..0ad3e40 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -58,6 +58,7 @@ private Token amRmToken = null; private Set applicationTags = null; private Priority priority = null; + private String appReportSource = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -630,4 +631,23 @@ public void setPriority(Priority priority) { builder.clearPriority(); this.priority = priority; } + + @Override + public String getAppReportSource() { + if (appReportSource != null) { + return this.appReportSource; + } + return null; + } + + @Override + public void setAppReportSource(String appReportSource) { + if (appReportSource != null && + (appReportSource == ApplicationReport.APP_REPORT_SOURCE_RM || + appReportSource == ApplicationReport.APP_REPORT_SOURCE_AHS)) { + this.appReportSource = appReportSource; + } else { + this.appReportSource = null; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java index 6aa43eb..8651485 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java @@ -121,9 +121,13 @@ public ApplicationReport getApplicationReport(ApplicationId appId) .newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(appId); - GetApplicationReportResponse response; + ApplicationReport appReport; try { - response = applicationsManager.getApplicationReport(request); + appReport = applicationsManager. + getApplicationReport(request).getApplicationReport(); + if (appReport != null) { + appReport.setAppReportSource(ApplicationReport.APP_REPORT_SOURCE_RM); + } } catch (YarnException e) { if (!isAHSEnabled) { // Just throw it as usual if historyService is not enabled. @@ -134,9 +138,13 @@ public ApplicationReport getApplicationReport(ApplicationId appId) if (!(e.getClass() == ApplicationNotFoundException.class)) { throw e; } - response = historyManager.getApplicationReport(request); + appReport = historyManager. + getApplicationReport(request).getApplicationReport(); + if (appReport != null) { + appReport.setAppReportSource(ApplicationReport.APP_REPORT_SOURCE_AHS); + } } - return response.getApplicationReport(); + return appReport; } public void stop() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 33f36f0..c5f2bae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -90,6 +90,7 @@ private transient List trackingUriPlugins; private final String rmAppPageUrlBase; + private final String ahsAppPageUrlBase; private transient YarnConfiguration conf; /** @@ -125,6 +126,9 @@ public WebAppProxyServlet() { TrackingUriPlugin.class); this.rmAppPageUrlBase = StringHelper.pjoin( WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app"); + this.ahsAppPageUrlBase = StringHelper.pjoin( + WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils + .getAHSWebAppURLWithoutScheme(conf), "applicationhistory", "apps"); } /** @@ -364,15 +368,28 @@ private void methodAction(final HttpServletRequest req, } notFound(resp, "Application " + appId + " could not be found, " + - "please try the history server"); + "in RM or history server"); return; } String original = applicationReport.getOriginalTrackingUrl(); URI trackingUri; - // fallback to ResourceManager's app page if no tracking URI provided - if(original == null || original.equals("N/A")) { - ProxyUtils.sendRedirect(req, resp, - StringHelper.pjoin(rmAppPageUrlBase, id.toString())); + if(original == null || original.equals("N/A") || original.equals("")) { + if(applicationReport.getAppReportSource() == + ApplicationReport.APP_REPORT_SOURCE_RM) { + // fallback to ResourceManager's app page if no tracking URI provided + // and Application Report was fetched from RM + LOG.debug("Original tracking url is '{}'. Redirecting to RM app page", + original == null? "NULL" : original); + ProxyUtils.sendRedirect(req, resp, + StringHelper.pjoin(rmAppPageUrlBase, id.toString())); + } else if(applicationReport.getAppReportSource() == + ApplicationReport.APP_REPORT_SOURCE_AHS) { + // fallback to Application History Server app page otherwise + LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page", + original == null? "NULL" : original); + ProxyUtils.sendRedirect(req, resp, + StringHelper.pjoin(ahsAppPageUrlBase, id.toString())); + } return; } else { if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 8e68c38..40b8f4e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.ConnectException; import java.net.HttpCookie; import java.net.HttpURLConnection; import java.net.URI; @@ -76,6 +77,7 @@ private static int numberOfHeaders = 0; private static final String UNKNOWN_HEADER = "Unknown-Header"; private static boolean hasUnknownHeader = false; + Configuration configuration = new Configuration(); /** @@ -137,8 +139,6 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) @Test(timeout=5000) public void testWebAppProxyServlet() throws Exception { - - Configuration configuration = new Configuration(); configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS configuration.setInt("hadoop.http.max.threads", 5); @@ -166,6 +166,7 @@ public void testWebAppProxyServlet() throws Exception { proxyConn.connect(); assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, proxyConn.getResponseCode()); + // set true Application ID in url URL url = new URL("http://localhost:" + proxyPort + "/proxy/application_00_0"); proxyConn = (HttpURLConnection) url.openConnection(); @@ -221,6 +222,49 @@ public void testWebAppProxyServlet() throws Exception { assertEquals("http://localhost:" + originalPort + "/foo/bar/test/tez?a=b&x=y&h=p#main", proxyConn.getURL().toString()); + // get appReport for empty tracking url + //set AHS_ENBALED = false to simulate getting the app report from RM + configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + false); + ApplicationId app = ApplicationId.newInstance(0, 0); + appReportFetcher.answer = 6; + url = new URL("http://localhost:" + proxyPort + + "/proxy/" + app.toString()); + proxyConn = (HttpURLConnection) url.openConnection(); + proxyConn.connect(); + try { + proxyConn.getResponseCode(); + } catch (ConnectException e) { + // Connection Exception is expected as we have set + // appReportFetcher.answer = 6, which does not set anything for + // original tracking url field in the app report. + } + //get the tracking url in RM + String appAddressInRm = + WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) + + "/cluster" + "/app/" + app.toString(); + assertTrue("Webapp proxy servlet should have redirected to RM", + proxyConn.getURL().toString().equals(appAddressInRm)); + + //set AHS_ENBALED = true to simulate getting the app report from AHS + configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + true); + proxyConn = (HttpURLConnection) url.openConnection(); + proxyConn.connect(); + try { + proxyConn.getResponseCode(); + } catch (ConnectException e) { + // Connection Exception is expected as we have set + // appReportFetcher.answer = 6, which does not set anything for + // original tracking url field in the app report. + } + //get the tracking url in AHS + String appAddressInAhs = WebAppUtils.getHttpSchemePrefix(configuration) + WebAppUtils + .getAHSWebAppURLWithoutScheme(configuration) + "/applicationhistory" + + "/apps/" + app.toString(); + assertTrue("Webapp proxy servlet should have redirected to AHS", + proxyConn.getURL().toString().equals(appAddressInAhs)); + } finally { proxy.close(); } @@ -398,7 +442,6 @@ protected void serviceStart() throws Exception { } private class AppReportFetcherForTest extends AppReportFetcher { - int answer = 0; public AppReportFetcherForTest(Configuration conf) { @@ -429,18 +472,37 @@ public ApplicationReport getApplicationReport(ApplicationId appId) + "/foo/bar?a=b#main"); result.setYarnApplicationState(YarnApplicationState.FINISHED); return result; + } else if (answer == 6) { + return getDefaultApplicationReport(appId, false); } return null; } - private ApplicationReport getDefaultApplicationReport(ApplicationId appId) { + /* + * If this method is called with isTrackingUrl=false, no tracking url + * will set in the app report. Hence, there will be a connection exception + * when the prxyCon tries to connect. + */ + private ApplicationReport getDefaultApplicationReport(ApplicationId appId, + boolean isTrackingUrl) { ApplicationReport result = new ApplicationReportPBImpl(); result.setApplicationId(appId); - result.setOriginalTrackingUrl("localhost:" + originalPort + "/foo/bar"); + if (isTrackingUrl) { + result.setOriginalTrackingUrl("localhost:" + originalPort + "/foo/bar"); + } else { + if(configuration.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, false)) { + result.setAppReportSource(ApplicationReport.APP_REPORT_SOURCE_AHS); + } else { + result.setAppReportSource(ApplicationReport.APP_REPORT_SOURCE_RM); + } + } result.setYarnApplicationState(YarnApplicationState.RUNNING); result.setUser(CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER); return result; } - + + private ApplicationReport getDefaultApplicationReport(ApplicationId appId) { + return getDefaultApplicationReport(appId, true); + } } }