diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 13c1d5e..5d1da44 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -226,6 +226,8 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, * The container ID * @param nmId * The Node Manager NodeId + * @param redirect + * Whether this is a redirect request * @return * The log file's name and current file size */ @@ -236,7 +238,8 @@ public Response getContainerLogsInfo( @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECT) String redirect) { ContainerId containerId = null; init(res); try { @@ -244,6 +247,11 @@ public Response getContainerLogsInfo( } catch (IllegalArgumentException e) { throw new BadRequestException("invalid container id, " + containerIdStr); } + boolean redirectRequest = false; + if (redirect != null && !redirect.isEmpty() && redirect.trim() + .toLowerCase().equals("true")) { + redirectRequest = true; + } ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); AppInfo appInfo; @@ -288,9 +296,12 @@ public Response getContainerLogsInfo( // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get log meta for aggregated logs instead of // re-directing the request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirectRequest) { // return log meta for the aggregated logs if exists. // It will also return empty log meta for the local logs. + // If this is the redirect request, we should not re-direct the + // request back. Simply output the aggregated log meta. return getContainerLogMeta(appId, appOwner, null, containerIdStr, true); } @@ -329,6 +340,8 @@ public Response getContainerLogsInfo( * the size of the log file * @param nmId * The Node Manager NodeId + * @param redirect + * Whether this is the redirect request * @return * The contents of the container's log file */ @@ -343,9 +356,10 @@ public Response getContainerLogFile(@Context HttpServletRequest req, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECT) String redirect) { return getLogs(req, res, containerIdStr, filename, format, - size, nmId); + size, nmId, redirect); } //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and @@ -362,8 +376,14 @@ public Response getLogs(@Context HttpServletRequest req, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECT) String redirect) { init(res); + boolean redirectRequest = false; + if (redirect != null && !redirect.isEmpty() && redirect.trim() + .toLowerCase().equals("true")) { + redirectRequest = true; + } ContainerId containerId; try { containerId = ContainerId.fromString(containerIdStr); @@ -417,8 +437,11 @@ public Response getLogs(@Context HttpServletRequest req, nodeHttpAddress = containerInfo.getNodeHttpAddress(); // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get aggregated logs instead of re-directing the - // request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + // request. + // If this is the redirect request, we should not re-direct the + // request back. Simply output the aggregated logs. + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirectRequest) { // output the aggregated logs return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, true); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 0f019dc..a5601a2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -767,6 +767,23 @@ public void testContainerLogsForRunningApps() throws Exception { assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL)); assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning())); + + // If this is the redirect request, we would not re-direct the request + // back and get the aggregated logs. + String content1 = "Hello." + containerId1; + NodeId nodeId1 = NodeId.fromString(NM_ID); + TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, + rootLogDir, containerId1, nodeId1, fileName, user, content1, true); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs").path(fileName) + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.REDIRECT, "true") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains(content1)); + assertTrue(responseText.contains("LogAggregationType: " + + ContainerLogAggregationType.AGGREGATED)); } @Test(timeout = 10000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 87e540f..a5cafed 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -34,4 +34,5 @@ String RESPONSE_CONTENT_FORMAT = "format"; String RESPONSE_CONTENT_SIZE = "size"; String NM_ID = "nm.id"; + String REDIRECT = "redirect"; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 655dc6b..411a221 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -491,7 +491,10 @@ private Response createRedirectResponse(HttpServletRequest httpRequest, String requestParams = WebAppUtils.removeQueryParams(httpRequest, YarnWebServiceParams.NM_ID); if (requestParams != null && !requestParams.isEmpty()) { - redirectPath.append("?" + requestParams); + redirectPath.append("?" + requestParams + "&" + + YarnWebServiceParams.REDIRECT + "=true"); + } else { + redirectPath.append("?" + YarnWebServiceParams.REDIRECT + "=true"); } ResponseBuilder res = Response.status( HttpServletResponse.SC_TEMPORARY_REDIRECT); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 68a6827..a2e1c0c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -376,6 +376,7 @@ public void testNMRedirect() { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains(YarnWebServiceParams.REDIRECT + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); // check the new api @@ -390,6 +391,7 @@ public void testNMRedirect() { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains(YarnWebServiceParams.REDIRECT + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); requestURI = r.path("ws").path("v1").path("node") @@ -402,6 +404,7 @@ public void testNMRedirect() { assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains(YarnWebServiceParams.REDIRECT + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); }