From d99e8ab8ab0f674b4887a013790c731fd5fcbc19 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Mon, 27 Jan 2020 17:56:20 +0100 Subject: [PATCH] YARN-10101. Support listing of aggregated logs for containers belonging to an application attempt --- .../mapreduce/v2/hs/webapp/HsWebServices.java | 26 ++- .../v2/hs/webapp/TestHsWebServices.java | 1 - .../v2/hs/webapp/TestHsWebServicesLogs.java | 30 +++ .../hadoop/yarn/client/cli/LogsCLI.java | 2 +- .../logaggregation/ContainerLogsRequest.java | 15 +- .../LogAggregationFileController.java | 12 ++ .../LogAggregationIndexedFileController.java | 12 +- .../tfile/LogAggregationTFileController.java | 9 +- .../webapp/AHSWebServices.java | 8 +- .../hadoop/yarn/server/webapp/LogServlet.java | 182 +++++++++++++++--- .../yarn/server/webapp/LogWebService.java | 7 +- .../server/webapp/LogWebServiceUtils.java | 50 ----- .../server/webapp/WrappedLogMetaRequest.java | 160 +++++++++++++++ .../server/webapp/YarnWebServiceParams.java | 2 + .../server/webapp/dao/ContainerLogsInfo.java | 2 +- 15 files changed, 427 insertions(+), 91 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WrappedLogMetaRequest.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index e3804e97b2d..bc6b57646dc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -69,6 +69,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.webapp.WrappedLogMetaRequest; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.LogServlet; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -422,18 +423,37 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + @GET + @Path("/aggregatedlogs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getAggregatedLogs(@Context HttpServletRequest hsr, + @QueryParam(YarnWebServiceParams.APP_ID) String appIdStr, + @QueryParam(YarnWebServiceParams.APPATTEMPT_ID) String appAttemptIdStr, + @QueryParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr) { + init(); + return logServlet.getLogsInfo(hsr, appIdStr, appAttemptIdStr, + containerIdStr); + } + @GET @Path("/containers/{containerid}/logs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @InterfaceAudience.Public @InterfaceStability.Unstable - public Response getLogs(@Context HttpServletRequest hsr, + public Response getContainerLogs(@Context HttpServletRequest hsr, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, @QueryParam(YarnWebServiceParams.NM_ID) String nmId, @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) @DefaultValue("false") boolean redirectedFromNode) { init(); - return logServlet.getContainerLogsInfo(hsr, containerIdStr, nmId, + + WrappedLogMetaRequest.Builder logMetaRequestBuilder = + WrappedLogMetaRequest.builder(); + logMetaRequestBuilder.setContainerId(containerIdStr); + + return logServlet.getContainerLogsInfo(hsr, logMetaRequestBuilder, nmId, redirectedFromNode, null); } @@ -442,7 +462,7 @@ public Response getLogs(@Context HttpServletRequest hsr, @Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 }) @InterfaceAudience.Public @InterfaceStability.Unstable - public Response getLogs(@Context HttpServletRequest req, + public Response getContainerLogFile(@Context HttpServletRequest req, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java index b4a4566bb53..e048e4734ee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java @@ -301,5 +301,4 @@ public void verifyHSInfoXML(String xml, AppContext ctx) WebServicesTestUtils.getXmlLong(element, "startedOn")); } } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java new file mode 100644 index 00000000000..124669e4305 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.v2.hs.webapp; + +public class TestHsWebServicesLogs { + // getAggregatedLogs for running containers? + // getAggregatedLogs for finished containers? + + // getContainerLogs for running containers? + // getContainerLogs for finished containers? + + // getContainerLogFile for running containers? + // getContainerLogFile for finished containers? + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index d52f6ad23a7..343dfc7bd2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -356,7 +356,7 @@ private int runCommand(String[] args) throws Exception { } - ContainerLogsRequest request = new ContainerLogsRequest(appId, + ContainerLogsRequest request = new ContainerLogsRequest(appId, null, Apps.isApplicationFinalState(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java index 30aeb6cc78b..defc0767a23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.logaggregation; import java.util.Set; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerState; public class ContainerLogsRequest { private ApplicationId appId; + private ApplicationAttemptId attemptId; private String containerId; private String nodeId; private String nodeHttpAddress; @@ -38,6 +41,7 @@ public ContainerLogsRequest() {} public ContainerLogsRequest(ContainerLogsRequest request) { this.setAppId(request.getAppId()); + this.setApplicationAttemptId(request.getAppAttemptId()); this.setAppFinished(request.isAppFinished()); this.setAppOwner(request.getAppOwner()); this.setNodeId(request.getNodeId()); @@ -50,10 +54,11 @@ public ContainerLogsRequest(ContainerLogsRequest request) { } public ContainerLogsRequest(ApplicationId applicationId, - boolean isAppFinished, String owner, + ApplicationAttemptId appAttemptId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, Set logs, long bytes, ContainerState containerState) { this.setAppId(applicationId); + this.setApplicationAttemptId(appAttemptId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); this.setNodeId(address); @@ -73,6 +78,14 @@ public void setAppId(ApplicationId appId) { this.appId = appId; } + public ApplicationAttemptId getAppAttemptId() { + return this.attemptId; + } + + public void setApplicationAttemptId(ApplicationAttemptId appAttemptId) { + this.attemptId = appAttemptId; + } + public String getContainerId() { return containerId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index ec633d6de41..824bbf2e9c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -49,7 +49,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -594,4 +596,14 @@ protected String aggregatedLogSuffix(String fileName) { public boolean isFsSupportsChmod() { return fsSupportsChmod; } + + protected boolean belongsToAppAttempt(ApplicationAttemptId appAttemptId, + String containerIdStr) { + try { + ContainerId containerId = ContainerId.fromString(containerIdStr); + return containerId.getApplicationAttemptId().equals(appAttemptId); + } catch (Exception ignore) { + } + return false; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 605997f3ab7..02c319eed1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -71,6 +71,7 @@ import org.apache.hadoop.io.file.tfile.Compression.Algorithm; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; @@ -619,8 +620,9 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, String nodeId = logRequest.getNodeId(); ApplicationId appId = logRequest.getAppId(); String appOwner = logRequest.getAppOwner(); - boolean getAllContainers = (containerIdStr == null || - containerIdStr.isEmpty()); + ApplicationAttemptId appAttemptId = logRequest.getAppAttemptId(); + boolean getAllContainers = ((containerIdStr == null || + containerIdStr.isEmpty()) && appAttemptId != null); String nodeIdStr = (nodeId == null || nodeId.isEmpty()) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator nodeFiles = LogAggregationUtils @@ -664,8 +666,12 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, if (getAllContainers) { for (Entry> log : logMeta .getLogMetas().entrySet()) { + String currentContainerIdStr = log.getKey(); + if (!belongsToAppAttempt(appAttemptId, currentContainerIdStr)) { + continue; + } ContainerLogMeta meta = new ContainerLogMeta( - log.getKey().toString(), curNodeId); + log.getKey(), curNodeId); for (IndexedFileLogMeta aMeta : log.getValue()) { meta.addLogMeta(aMeta.getFileName(), Long.toString( aMeta.getFileSize()), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index 3fb432d7737..cc0ac0b43c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -27,6 +27,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.math3.util.Pair; @@ -264,7 +266,9 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, String nodeId = logRequest.getNodeId(); ApplicationId appId = logRequest.getAppId(); String appOwner = logRequest.getAppOwner(); - boolean getAllContainers = (containerIdStr == null); + ApplicationAttemptId appAttemptId = logRequest.getAppAttemptId(); + boolean getAllContainers = (containerIdStr == null && + appAttemptId == null); String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator nodeFiles = LogAggregationUtils @@ -297,7 +301,8 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, LogKey key = new LogKey(); valueStream = reader.next(key); while (valueStream != null) { - if (getAllContainers || (key.toString().equals(containerIdStr))) { + if (getAllContainers || (key.toString().equals(containerIdStr)) || + belongsToAppAttempt(appAttemptId, key.toString())) { ContainerLogMeta containerLogMeta = new ContainerLogMeta( key.toString(), thisNodeFile.getPath().getName()); while (true) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 5e77718102a..4f52c9d3094 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.server.webapp.LogServlet; import org.apache.hadoop.yarn.server.webapp.WebServices; +import org.apache.hadoop.yarn.server.webapp.WrappedLogMetaRequest; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo; @@ -225,7 +226,12 @@ public Response getContainerLogsInfo( @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) @DefaultValue("false") boolean redirected_from_node) { initForReadableEndpoints(res); - return logServlet.getContainerLogsInfo(req, containerIdStr, nmId, + + WrappedLogMetaRequest.Builder logMetaRequestBuilder = + WrappedLogMetaRequest.builder(); + logMetaRequestBuilder.setContainerId(containerIdStr); + + return logServlet.getContainerLogsInfo(req, logMetaRequestBuilder, nmId, redirected_from_node, null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 39e0ae308b5..a17b7dc1ab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -24,11 +24,14 @@ import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.util.Apps; -import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; @@ -36,8 +39,12 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.GenericEntity; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import java.util.ArrayList; +import java.util.List; /** * Extracts aggregated logs and related information. @@ -65,46 +72,166 @@ public String getNMWebAddressFromRM(String nodeId) return LogWebServiceUtils.getNMWebAddressFromRM(getConf(), nodeId); } + private static List convertToContainerLogsInfo( + List containerLogMetas, + boolean emptyLocalContainerLogMeta) { + List containersLogsInfo = new ArrayList<>(); + List containerIds = new ArrayList<>(); + for (ContainerLogMeta meta : containerLogMetas) { + ContainerLogsInfo logInfo = + new ContainerLogsInfo(meta, ContainerLogAggregationType.AGGREGATED); + containersLogsInfo.add(logInfo); + containerIds.add(meta.getContainerId()); + } + if (emptyLocalContainerLogMeta) { + for (String containerId : containerIds) { + ContainerLogMeta emptyMeta = + new ContainerLogMeta(containerId, "N/A"); + ContainerLogsInfo empty = + new ContainerLogsInfo(emptyMeta, ContainerLogAggregationType.LOCAL); + containersLogsInfo.add(empty); + } + // TODO maybe order by containerId ? + } + return containersLogsInfo; + } + + private static Response getContainerLogMeta( + WrappedLogMetaRequest request, boolean emptyLocalContainerLogMeta) { + try { + List containerLogMeta = request.getContainerLogMetas(); + if (containerLogMeta.isEmpty()) { + throw new NotFoundException("Can not get log meta for request."); + } + List containersLogsInfo = convertToContainerLogsInfo( + containerLogMeta, emptyLocalContainerLogMeta); + + GenericEntity> meta = + new GenericEntity>(containersLogsInfo) { + }; + Response.ResponseBuilder response = Response.ok(meta); + // Sending the X-Content-Type-Options response header with the value + // nosniff will prevent Internet Explorer from MIME-sniffing a response + // away from the declared content-type. + response.header("X-Content-Type-Options", "nosniff"); + return response.build(); + } catch (Exception ex) { + throw new WebApplicationException(ex); + } + } + + /** + * Validates whether the user has provided at least one query param for + * the request. Also validates that if multiple query params are provided, + * they do not contradict. + */ + private void validateUserInput(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, ContainerId containerId) { + if (applicationId == null && applicationAttemptId == null && + containerId == null) { + throw new IllegalArgumentException("Should set application id, " + + "application attempt id or container id."); + } + + if (containerId != null) { + if (applicationAttemptId != null) { + if (!applicationAttemptId.equals(containerId + .getApplicationAttemptId())) { + throw new IllegalArgumentException( + String.format( + "Container %s does not belong to application attempt %s!", + containerId, applicationAttemptId)); + } + } + if (applicationId != null) { + if (!applicationId.equals(containerId.getApplicationAttemptId() + .getApplicationId())) { + throw new IllegalArgumentException( + String.format( + "Container %s does not belong to application %s!", + containerId, applicationId)); + } + } + } + if (applicationAttemptId != null) { + if (applicationId != null) { + if (!applicationId.equals(applicationAttemptId.getApplicationId())) { + throw new IllegalArgumentException( + String.format( + "Application attempt %s does not belong to application %s!", + applicationAttemptId, applicationId)); + } + } + } + } + + public Response getLogsInfo(HttpServletRequest hsr, String appIdStr, + String appAttemptIdStr, String containerIdStr) { + ApplicationId appId = null; + try { + appId = ApplicationId.fromString(appIdStr); + } catch (Exception ignore) { + } + + ApplicationAttemptId appAttemptId = null; + try { + appAttemptId = ApplicationAttemptId + .fromString(appAttemptIdStr); + } catch (Exception ignore) { + } + + ContainerId containerId = null; + try { + containerId = ContainerId.fromString(containerIdStr); + } catch (Exception ignore) { + } + + validateUserInput(appId, appAttemptId, containerId); + + WrappedLogMetaRequest.Builder logMetaRequestBuilder = + WrappedLogMetaRequest.builder() + .setApplicationId(appId) + .setApplicationAttemptId(appAttemptId) + .setContainerId(containerIdStr); + + return getContainerLogsInfo(hsr, logMetaRequestBuilder, null, + false, null); + } + /** * Returns information about the logs for a specific container. * * @param req the {@link HttpServletRequest} - * @param containerIdStr container id + * @param builder builder instance for the log meta request * @param nmId NodeManager id * @param redirectedFromNode whether the request was redirected * @param clusterId the id of the cluster * @return {@link Response} object containing information about the logs */ public Response getContainerLogsInfo(HttpServletRequest req, - String containerIdStr, String nmId, boolean redirectedFromNode, + WrappedLogMetaRequest.Builder builder, + String nmId, boolean redirectedFromNode, String clusterId) { - ContainerId containerId = null; - try { - containerId = ContainerId.fromString(containerIdStr); - } catch (IllegalArgumentException e) { - throw new BadRequestException("invalid container id, " + containerIdStr); - } - ApplicationId appId = containerId.getApplicationAttemptId() - .getApplicationId(); + builder.setFactory(factory); + BasicAppInfo appInfo; try { - appInfo = appInfoProvider.getApp(req, appId.toString(), clusterId); + appInfo = appInfoProvider.getApp(req, builder.getAppId(), clusterId); } catch (Exception ex) { // directly find logs from HDFS. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); + return getContainerLogMeta(builder.build(), false); } // if the application finishes, directly find logs // from HDFS. if (Apps.isApplicationFinalState(appInfo.getAppState())) { - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); + return getContainerLogMeta(builder.build(), false); } if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) { String appOwner = appInfo.getUser(); + builder.setAppOwner(appOwner); + WrappedLogMetaRequest request = builder.build(); + String nodeHttpAddress = null; if (nmId != null && !nmId.isEmpty()) { try { @@ -116,15 +243,12 @@ public Response getContainerLogsInfo(HttpServletRequest req, if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { try { nodeHttpAddress = appInfoProvider.getNodeHttpAddress( - req, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString(), clusterId); + req, request.getAppId(), request.getAppAttemptId(), + request.getContainerId().toString(), clusterId); } catch (Exception ex) { // return log meta for the aggregated logs if exists. // It will also return empty log meta for the local logs. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); + return getContainerLogMeta(request, true); } // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get log meta for aggregated logs instead of @@ -135,11 +259,15 @@ public Response getContainerLogsInfo(HttpServletRequest req, // It will also return empty log meta for the local logs. // If this is the redirect request from NM, we should not // re-direct the request back. Simply output the aggregated log meta. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); + return getContainerLogMeta(request, true); } } + ContainerId containerId = request.getContainerId(); + if (containerId == null) { + throw new WebApplicationException( + new Exception("Could not redirect to node, as app attempt or " + + "application logs are requested.")); + } String uri = "/" + containerId.toString() + "/logs"; String resURI = JOINER.join( LogWebServiceUtils.getAbsoluteNMWebAddress(getConf(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java index 1ad6b61e9be..f3a46f87bee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java @@ -151,7 +151,12 @@ public Response getContainerLogsInfo(@Context HttpServletRequest req, @DefaultValue("false") boolean redirectedFromNode, @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) { initForReadableEndpoints(res); - return logServlet.getContainerLogsInfo(req, containerIdStr, nmId, + + WrappedLogMetaRequest.Builder logMetaRequestBuilder = + WrappedLogMetaRequest.builder(); + logMetaRequestBuilder.setContainerId(containerIdStr); + + return logServlet.getContainerLogsInfo(req, logMetaRequestBuilder, nmId, redirectedFromNode, clusterId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java index defde4d39aa..5c4838579b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java @@ -29,12 +29,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; -import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; -import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.webapp.ForbiddenException; -import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONException; @@ -42,16 +39,13 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.GenericEntity; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.UndeclaredThrowableException; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Set; /** @@ -66,50 +60,6 @@ private LogWebServiceUtils() { private static final Joiner DOT_JOINER = Joiner.on(". "); - public static Response getContainerLogMeta( - LogAggregationFileControllerFactory factory, ApplicationId appId, - String appOwner, final String nodeId, final String containerIdStr, - boolean emptyLocalContainerLogMeta) { - try { - ContainerLogsRequest request = new ContainerLogsRequest(); - request.setAppId(appId); - request.setAppOwner(appOwner); - request.setContainerId(containerIdStr); - request.setNodeId(nodeId); - List containerLogMeta = - factory.getFileControllerForRead(appId, appOwner) - .readAggregatedLogsMeta(request); - if (containerLogMeta.isEmpty()) { - throw new NotFoundException( - "Can not get log meta for container: " + containerIdStr); - } - List containersLogsInfo = new ArrayList<>(); - for (ContainerLogMeta meta : containerLogMeta) { - ContainerLogsInfo logInfo = - new ContainerLogsInfo(meta, ContainerLogAggregationType.AGGREGATED); - containersLogsInfo.add(logInfo); - } - if (emptyLocalContainerLogMeta) { - ContainerLogMeta emptyMeta = - new ContainerLogMeta(containerIdStr, "N/A"); - ContainerLogsInfo empty = - new ContainerLogsInfo(emptyMeta, ContainerLogAggregationType.LOCAL); - containersLogsInfo.add(empty); - } - GenericEntity> meta = - new GenericEntity>(containersLogsInfo) { - }; - Response.ResponseBuilder response = Response.ok(meta); - // Sending the X-Content-Type-Options response header with the value - // nosniff will prevent Internet Explorer from MIME-sniffing a response - // away from the declared content-type. - response.header("X-Content-Type-Options", "nosniff"); - return response.build(); - } catch (Exception ex) { - throw new WebApplicationException(ex); - } - } - public static Response sendStreamOutputResponse( LogAggregationFileControllerFactory factory, ApplicationId appId, String appOwner, String nodeId, String containerIdStr, String fileName, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WrappedLogMetaRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WrappedLogMetaRequest.java new file mode 100644 index 00000000000..01829f550ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WrappedLogMetaRequest.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.List; + +/** + * WrappedLogMetaRequest is wrapping a log request initiated by the client. + * This wrapper class translates the request to a {@link ContainerLogsRequest} + * and calls #readAggregatedLogsMeta on the + * {@link org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController} + * class. + */ +public class WrappedLogMetaRequest { + + private final LogAggregationFileControllerFactory factory; + private final ApplicationId appId; + private final String appOwner; + private final ContainerId containerId; + private final String nodeId; + private final ApplicationAttemptId applicationAttemptId; + + private WrappedLogMetaRequest(Builder builder) { + this.factory = builder.factory; + this.appId = builder.appId; + this.appOwner = builder.appOwner; + this.containerId = builder.containerId; + this.nodeId = builder.nodeId; + this.applicationAttemptId = builder.applicationAttemptId; + } + + public static class Builder { + private LogAggregationFileControllerFactory factory; + private ApplicationId appId; + private String appOwner; + private ContainerId containerId; + private String nodeId; + private ApplicationAttemptId applicationAttemptId; + + Builder() { + } + + Builder setFactory(LogAggregationFileControllerFactory logFactory) { + this.factory = logFactory; + return this; + } + + public Builder setApplicationId(ApplicationId applicationId) { + this.appId = applicationId; + return this; + } + + Builder setNodeId(String nid) { + this.nodeId = nid; + return this; + } + + public Builder setContainerId(@Nullable String containerIdStr) { + if (containerIdStr != null) { + this.containerId = ContainerId.fromString(containerIdStr); + } + return this; + } + + Builder setAppOwner(String user) { + this.appOwner = user; + return this; + } + + public Builder setApplicationAttemptId(ApplicationAttemptId + applicationAttemptId) { + this.applicationAttemptId = applicationAttemptId; + return this; + } + + String getAppId() { + return WrappedLogMetaRequest.getAppId(appId, applicationAttemptId, + containerId); + } + + WrappedLogMetaRequest build() { + if (this.factory == null) { + throw new AssertionError("WrappedLogMetaRequest's builder should be " + + "given a LogAggregationFileControllerFactory as parameter."); + } + return new WrappedLogMetaRequest(this); + } + } + + public static Builder builder() { + return new Builder(); + } + + private static String getAppId(ApplicationId appId, + ApplicationAttemptId applicationAttemptId, ContainerId containerId) { + if (appId == null) { + if (applicationAttemptId == null) { + return containerId.getApplicationAttemptId().getApplicationId() + .toString(); + } else { + return applicationAttemptId.getApplicationId().toString(); + } + } + return appId.toString(); + } + + public String getAppId() { + return getAppId(appId, applicationAttemptId, containerId); + } + + public String getAppAttemptId() { + if (applicationAttemptId == null) { + return containerId.getApplicationAttemptId().toString(); + } else { + return applicationAttemptId.toString(); + } + } + + public ContainerId getContainerId() { + return containerId; + } + + public List getContainerLogMetas() throws IOException { + ApplicationId applicationId = ApplicationId.fromString(getAppId()); + + ContainerLogsRequest request = new ContainerLogsRequest(); + request.setAppId(applicationId); + request.setApplicationAttemptId(applicationAttemptId); + request.setContainerId(containerId.toString()); + request.setAppOwner(appOwner); + request.setNodeId(nodeId); + + return factory.getFileControllerForRead(applicationId, appOwner) + .readAggregatedLogsMeta(request); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 5f96f231524..c737fc82a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -29,6 +29,8 @@ public interface YarnWebServiceParams { // the params used in container-log related web services + String APP_ID = "appid"; + String APPATTEMPT_ID = "appattemptid"; String CONTAINER_ID = "containerid"; String CONTAINER_LOG_FILE_NAME = "filename"; String RESPONSE_CONTENT_FORMAT = "format"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java index 1bb0408d944..d27c4829b14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerLogsInfo.java @@ -61,7 +61,7 @@ public ContainerLogsInfo() {} public ContainerLogsInfo(ContainerLogMeta logMeta, - ContainerLogAggregationType logType) throws YarnException { + ContainerLogAggregationType logType) { this.containerLogsInfo = new ArrayList( logMeta.getContainerLogMeta()); this.logType = logType.toString(); -- 2.21.0