diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index d2dbbba..2089214 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogType; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; +import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; @@ -215,6 +216,8 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, * HttpServletResponse * @param containerIdStr * The container ID + * @param nmWebAddress + * The Node Manager Web Address * @return * The log file's name and current file size */ @@ -224,7 +227,8 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, public Response getContainerLogsInfo( @Context HttpServletRequest req, @Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr) { + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @QueryParam(YarnWebServiceParams.NM_WEB_ADDRESS) String nmWebAddress) { ContainerId containerId = null; init(res); try { @@ -249,21 +253,25 @@ public Response getContainerLogsInfo( } if (isRunningState(appInfo.getAppState())) { String appOwner = appInfo.getUser(); - ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - return getContainerLogMeta(appId, appOwner, null, - containerIdStr, true); + String nodeHttpAddress = nmWebAddress; + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + ContainerInfo containerInfo; + try { + containerInfo = super.getContainer( + req, res, appId.toString(), + containerId.getApplicationAttemptId().toString(), + containerId.toString()); + } catch (Exception ex) { + // return log meta for the aggregated logs if exists. + // It will also return empty log meta for the local logs. + return getContainerLogMeta(appId, appOwner, null, + containerIdStr, true); + } + nodeHttpAddress = containerInfo.getNodeHttpAddress(); } - String nodeHttpAddress = containerInfo.getNodeHttpAddress(); String uri = "/" + containerId.toString() + "/logs"; - String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); + String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress), + NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { resURI += "?" + query; @@ -293,6 +301,8 @@ public Response getContainerLogsInfo( * The content type * @param size * the size of the log file + * @param nmWebAddress + * The Node Manager Web Address * @return * The contents of the container's log file */ @@ -303,11 +313,13 @@ public Response getContainerLogsInfo( @Unstable public Response getContainerLogFile(@Context HttpServletRequest req, @Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("format") String format, - @QueryParam("size") String size) { - return getLogs(req, res, containerIdStr, filename, format, size); + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, + @QueryParam(YarnWebServiceParams.NM_WEB_ADDRESS) String nmWebAddress) { + return getLogs(req, res, containerIdStr, filename, format, + size, nmWebAddress); } //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and @@ -320,10 +332,11 @@ public Response getContainerLogFile(@Context HttpServletRequest req, @Unstable public Response getLogs(@Context HttpServletRequest req, @Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("format") String format, - @QueryParam("size") String size) { + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, + @QueryParam(YarnWebServiceParams.NM_WEB_ADDRESS) String nmWebAddress) { init(res); ContainerId containerId; try { @@ -353,20 +366,24 @@ public Response getLogs(@Context HttpServletRequest req, } if (isRunningState(appInfo.getAppState())) { + String nodeHttpAddress = nmWebAddress; + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - // output the aggregated logs - return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, format, length, true); + try { + containerInfo = super.getContainer( + req, res, appId.toString(), + containerId.getApplicationAttemptId().toString(), + containerId.toString()); + } catch (Exception ex) { + // output the aggregated logs + return sendStreamOutputResponse(appId, appOwner, null, + containerIdStr, filename, format, length, true); + } + nodeHttpAddress = containerInfo.getNodeHttpAddress(); } - String nodeHttpAddress = containerInfo.getNodeHttpAddress(); String uri = "/" + containerId.toString() + "/logs/" + filename; - String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); + String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress), + NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { resURI += "?" + query; @@ -477,7 +494,7 @@ private Response getContainerLogMeta(ApplicationId appId, String appOwner, .getContainerLogMetaFromRemoteFS(conf, appId, containerIdStr, nodeId, appOwner); if (containerLogMeta.isEmpty()) { - return createBadResponse(Status.INTERNAL_SERVER_ERROR, + throw new NotFoundException( "Can not get log meta for container: " + containerIdStr); } List containersLogsInfo = new ArrayList<>(); @@ -513,4 +530,12 @@ public static String getNoRedirectWarning() { + "re-direct the request to related NodeManager " + "for local container logs."; } + + private String getAbsoluteNMWebAddress(String nmWebAddress) { + if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) || + nmWebAddress.contains(WebAppUtils.HTTPS_PREFIX)) { + return nmWebAddress; + } + return WebAppUtils.getHttpSchemePrefix(conf) + nmWebAddress; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 935e140..9a5d0dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -672,6 +673,22 @@ public void testContainerLogsForRunningApps() throws Exception { assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); + // If we specify NM Web Address, we would re-direct the request + // to the specify NM Web Address. + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1.toString()).path(fileName) + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.NM_WEB_ADDRESS, "testnm:1234") + .getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); + assertTrue(redirectURL.contains("user.name=" + user)); + // Test with new API requestURI = r.path("ws").path("v1") .path("applicationhistory").path("containers") @@ -685,6 +702,20 @@ public void testContainerLogsForRunningApps() throws Exception { assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs").path(fileName) + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.NM_WEB_ADDRESS, "testnm:1234") + .getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); + assertTrue(redirectURL.contains("user.name=" + user)); + // If we can not container information from ATS, we would try to // get aggregated log from remote FileSystem. ContainerId containerId1000 = ContainerId.newContainerId( @@ -717,21 +748,38 @@ public void testContainerLogsMetaForRunningApps() throws Exception { ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); WebResource r = resource(); - // If we can get Container information from ATS, we re-direct the request - // to the nodemamager who runs the container. + // If we specify the NMWebAddress, we re-direct the request by using + // the specified NM web address URI requestURI = r.path("ws").path("v1") .path("applicationhistory").path("containers") .path(containerId1.toString()).path("logs") - .queryParam("user.name", user).getURI(); + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.NM_WEB_ADDRESS, "testnm:1234") + .getURI(); String redirectURL = getRedirectURL(requestURI.toString()); assertTrue(redirectURL != null); + assertTrue(redirectURL.contains("testnm:1234")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); + assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs")); + + // If we do not specify the NMWebAddress but can get Container information + // from ATS, we re-direct the request to the node manager + // who runs the container. + requestURI = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs") + .queryParam("user.name", user).getURI(); + redirectURL = getRedirectURL(requestURI.toString()); + assertTrue(redirectURL != null); assertTrue(redirectURL.contains("test:1234")); assertTrue(redirectURL.contains("ws/v1/node/containers")); assertTrue(redirectURL.contains(containerId1.toString())); assertTrue(redirectURL.contains("/logs")); - // If we can not container information from ATS, we would try to - // get aggregated log meta from remote FileSystem. + // If we can not container information from ATS, + // and not specify nmwebAddress, + // we would try to get aggregated log meta from remote FileSystem. ContainerId containerId1000 = ContainerId.newContainerId( appAttemptId, 1000); String fileName = "syslog"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java new file mode 100644 index 0000000..0f862dc --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -0,0 +1,37 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.webapp; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Common web service parameters which could be used in + * RM/NM/AHS WebService + * + */ +@InterfaceAudience.LimitedPrivate({"YARN"}) +public interface YarnWebServiceParams { + + // the params used in container-log related web services + String CONTAINER_ID = "containerid"; + String CONTAINER_LOG_FILE_NAME = "filename"; + String RESPONSE_CONTENT_FORMAT = "format"; + String RESPONSE_CONTENT_SIZE = "size"; + String NM_WEB_ADDRESS = "nmwebaddress"; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 7a9498b..22b85dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMContainerLogsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo; +import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -224,7 +225,7 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context public Response getContainerLogsInfo( @javax.ws.rs.core.Context HttpServletRequest hsr, @javax.ws.rs.core.Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr) { + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr) { ContainerId containerId = null; init(); try { @@ -295,10 +296,14 @@ public Response getContainerLogsInfo( @Public @Unstable public Response getContainerLogFile( - @PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("format") String format, - @QueryParam("size") String size) { + @PathParam(YarnWebServiceParams.CONTAINER_ID) + final String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) + String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) + String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) + String size) { return getLogs(containerIdStr, filename, format, size); } @@ -326,10 +331,14 @@ public Response getContainerLogFile( @Public @Unstable public Response getLogs( - @PathParam("containerid") final String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("format") String format, - @QueryParam("size") String size) { + @PathParam(YarnWebServiceParams.CONTAINER_ID) + final String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) + String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) + String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) + String size) { ContainerId tempContainerId; try { tempContainerId = ContainerId.fromString(containerIdStr);