diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 607b55a..5a2bd40 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -214,6 +214,8 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, * HttpServletResponse * @param containerIdStr * The container ID + * @param nmWebAddress + * The Node Manager Web Address * @return * The log file's name and current file size */ @@ -223,7 +225,8 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, public Response getContainerLogsInfo( @Context HttpServletRequest req, @Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr) { + @PathParam("containerid") String containerIdStr, + @QueryParam("nmwebaddress") String nmWebAddress) { ContainerId containerId = null; init(res); try { @@ -248,21 +251,25 @@ public Response getContainerLogsInfo( } if (isRunningState(appInfo.getAppState())) { String appOwner = appInfo.getUser(); - ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - return getContainerLogMeta(appId, appOwner, null, - containerIdStr, true); + String nodeHttpAddress = nmWebAddress; + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + ContainerInfo containerInfo; + try { + containerInfo = super.getContainer( + req, res, appId.toString(), + containerId.getApplicationAttemptId().toString(), + containerId.toString()); + } catch (Exception ex) { + // return log meta for the aggregated logs if exists. + // It will also return empty log meta for the local logs. + return getContainerLogMeta(appId, appOwner, null, + containerIdStr, true); + } + nodeHttpAddress = containerInfo.getNodeHttpAddress(); } - String nodeHttpAddress = containerInfo.getNodeHttpAddress(); String uri = "/" + containerId.toString() + "/logs"; - String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); + String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress), + NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { resURI += "?" + query; @@ -292,6 +299,8 @@ public Response getContainerLogsInfo( * The content type * @param size * the size of the log file + * @param nmWebAddress + * The Node Manager Web Address * @return * The contents of the container's log file */ @@ -305,8 +314,10 @@ public Response getContainerLogFile(@Context HttpServletRequest req, @PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, @QueryParam("format") String format, - @QueryParam("size") String size) { - return getLogs(req, res, containerIdStr, filename, format, size); + @QueryParam("size") String size, + @QueryParam("nmwebaddress") String nmWebAddress) { + return getLogs(req, res, containerIdStr, filename, format, + size, nmWebAddress); } //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and @@ -322,7 +333,8 @@ public Response getLogs(@Context HttpServletRequest req, @PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, @QueryParam("format") String format, - @QueryParam("size") String size) { + @QueryParam("size") String size, + @QueryParam("nmwebaddress") String nmWebAddress) { init(res); ContainerId containerId; try { @@ -352,20 +364,24 @@ public Response getLogs(@Context HttpServletRequest req, } if (isRunningState(appInfo.getAppState())) { + String nodeHttpAddress = nmWebAddress; + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - // output the aggregated logs - return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, format, length, true); + try { + containerInfo = super.getContainer( + req, res, appId.toString(), + containerId.getApplicationAttemptId().toString(), + containerId.toString()); + } catch (Exception ex) { + // output the aggregated logs + return sendStreamOutputResponse(appId, appOwner, null, + containerIdStr, filename, format, length, true); + } + nodeHttpAddress = containerInfo.getNodeHttpAddress(); } - String nodeHttpAddress = containerInfo.getNodeHttpAddress(); String uri = "/" + containerId.toString() + "/logs/" + filename; - String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); + String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress), + NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { resURI += "?" + query; @@ -513,4 +529,12 @@ public static String getNoRedirectWarning() { + "re-direct the request to related NodeManager " + "for local container logs."; } + + private String getAbsoluteNMWebAddress(String nmWebAddress) { + if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) || + nmWebAddress.contains(WebAppUtils.HTTPS_PREFIX)) { + return nmWebAddress; + } + return WebAppUtils.getHttpSchemePrefix(conf) + nmWebAddress; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 10ede37..88b52a4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -669,6 +669,21 @@ public void testContainerLogsForRunningApps() throws Exception { assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); + // If we specify NM Web Address, we would re-direct the request + // to the specify NM Web Address. + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1.toString()).path(fileName) + .queryParam("user.name", user) + .queryParam("nmwebaddress", "testnm:1234").getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); + assertTrue(redirectURL.contains("user.name=" + user)); + // Test with new API requestURI = r.path("ws").path("v1") .path("applicationhistory").path("containers") @@ -682,6 +697,19 @@ public void testContainerLogsForRunningApps() throws Exception { assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs").path(fileName) + .queryParam("user.name", user) + .queryParam("nmwebaddress", "testnm:1234").getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); + assertTrue(redirectURL.contains("user.name=" + user)); + // If we can not container information from ATS, we would try to // get aggregated log from remote FileSystem. ContainerId containerId1000 = ContainerId.newContainerId( @@ -714,21 +742,37 @@ public void testContainerLogsMetaForRunningApps() throws Exception { ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); WebResource r = resource(); - // If we can get Container information from ATS, we re-direct the request - // to the nodemamager who runs the container. + // If we specify the NMWebAddress, we re-direct the request by using + // the specified NM web address URI requestURI = r.path("ws").path("v1") .path("applicationhistory").path("containers") .path(containerId1.toString()).path("logs") - .queryParam("user.name", user).getURI(); + .queryParam("user.name", user) + .queryParam("nmwebaddress", "testnm:1234").getURI(); String redirectURL = getRedirectURL(requestURI.toString()); assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs")); + + // If we do not specify the NMWebAddress but can get Container information + // from ATS, we re-direct the request to the node manager + // who runs the container. + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs") + .queryParam("user.name", user).getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); assertTrue(redirectURL.contains("test:1234")); assertTrue(redirectURL.contains("ws/v1/node/containers")); assertTrue(redirectURL.contains(containerId1.toString())); assertTrue(redirectURL.contains("/logs")); - // If we can not container information from ATS, we would try to - // get aggregated log meta from remote FileSystem. + // If we can not container information from ATS, + // and not specify nmwebAddress, + // we would try to get aggregated log meta from remote FileSystem. ContainerId containerId1000 = ContainerId.newContainerId( appAttemptId, 1000); String fileName = "syslog";