diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 039ba3c..c3bdc0e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -20,16 +20,12 @@ import java.io.IOException; import java.io.PrintStream; -import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.ws.rs.core.MediaType; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -53,6 +49,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; +import org.apache.hadoop.yarn.logaggregation.LogsCLIOptions; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -66,9 +63,6 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; @Public @Evolving @@ -181,9 +175,19 @@ public int run(String[] args) throws Exception { } } - ContainerLogsRequest request = new ContainerLogsRequest(appId, + // if we did not specify logFiles option or specify "ALL", + // we will fetch all logs files for the container + List logs = new ArrayList(); + if (logFiles == null || logFiles.length == 0 + || fetchAllLogFiles(logFiles)) { + logs.add(".*"); + } else { + logs = Arrays.asList(logFiles); + } + + LogsCLIOptions request = new LogsCLIOptions(appId, isApplicationFinished(appState), appOwner, - nodeAddress, null, containerIdStr); + nodeAddress, null, containerIdStr, localDir, logs); if (showMetaInfo) { return showMetaInfo(request, logCliHelper); @@ -195,8 +199,7 @@ public int run(String[] args) throws Exception { // To get am logs if (getAMContainerLogs) { - return fetchAMContainerLogs(request, amContainersList, - logFiles, logCliHelper, localDir); + return fetchAMContainerLogs(request, amContainersList, logCliHelper); } int resultCode = 0; @@ -208,12 +211,10 @@ public int run(String[] args) throws Exception { + " does not have the container:" + containerId); return -1; } - return fetchContainerLogs(request, logFiles, - logCliHelper, localDir); + return fetchContainerLogs(request, logCliHelper); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(appId, appOwner, - logCliHelper, localDir); + resultCode = fetchApplicationLogs(request, logCliHelper); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -314,60 +315,17 @@ private boolean fetchAllLogFiles(String[] logFiles) { return false; } - private String[] getContainerLogFiles(Configuration conf, - String containerIdStr, String nodeHttpAddress) throws IOException { - List logFiles = new ArrayList<>(); - Client webServiceClient = Client.create(); - try { - WebResource webResource = webServiceClient - .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); - ClientResponse response = - webResource.path("ws").path("v1").path("node").path("containers") - .path(containerIdStr).accept(MediaType.APPLICATION_XML) - .get(ClientResponse.class); - if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) { - try { - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList elements = dom.getElementsByTagName("containerLogFiles"); - for (int i = 0; i < elements.getLength(); i++) { - logFiles.add(elements.item(i).getTextContent()); - } - } catch (Exception e) { - System.err.println("Unable to parse xml from webservice. Error:"); - System.err.println(e.getMessage()); - throw new IOException(e); - } - } - - } catch (ClientHandlerException | UniformInterfaceException ex) { - System.err.println("Unable to fetch log files list"); - throw new IOException(ex); - } - return logFiles.toArray(new String[0]); - } - private void printContainerLogsFromRunningApplication(Configuration conf, - ContainerLogsRequest request, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws IOException { - String appId = request.getAppId().toString(); + LogsCLIOptions request, LogCLIHelpers logCliHelper) + throws IOException { String containerIdStr = request.getContainerId().toString(); - String[] requestedLogFiles = logFiles; String nodeHttpAddress = request.getNodeHttpAddress(); String nodeId = request.getNodeId(); - String appOwner = request.getAppOwner(); + String localDir = request.getOutputLocalDir(); + List requestedLogFiles = request.getLogTypes(); PrintStream out = logCliHelper.createPrintStream(localDir, nodeId, containerIdStr); try { - // fetch all the log files for the container - if (fetchAllLogFiles(logFiles)) { - requestedLogFiles = - getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress); - } Client webServiceClient = Client.create(); String containerString = "\n\nContainer: " + containerIdStr; out.println(containerString); @@ -399,27 +357,19 @@ private void printContainerLogsFromRunningApplication(Configuration conf, logCliHelper.closePrintStream(out); } // for the case, we have already uploaded partial logs in HDFS - logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId, - appOwner, Arrays.asList(requestedLogFiles), false, localDir); + logCliHelper.dumpAContainersLogsForALogType(request, false); } - private void printContainerLogsForFinishedApplication( - ContainerLogsRequest request, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) + private int printContainerLogsForFinishedApplication( + LogsCLIOptions request, LogCLIHelpers logCliHelper) throws IOException { - logCliHelper.dumpAContainersLogsForALogType(request.getAppId().toString(), - request.getContainerId().toString(), request.getNodeId(), - request.getAppOwner(), logFiles != null ? Arrays.asList(logFiles) - : null, localDir); + return logCliHelper.dumpAContainersLogsForALogType(request); } private int printContainerLogsForFinishedApplicationWithoutNodeId( - String appId, String containerId, String[] logFiles, - LogCLIHelpers logCliHelper, String appOwner, String localDir) + LogsCLIOptions request, LogCLIHelpers logCliHelper) throws IOException { - return logCliHelper.dumpAContainersLogsForALogTypeWithoutNodeId(appId, - containerId, appOwner, logFiles != null ? - Arrays.asList(logFiles) : null, localDir); + return logCliHelper.dumpAContainersLogsForALogTypeWithoutNodeId(request); } private ContainerReport getContainerReport(String containerIdStr) @@ -439,13 +389,12 @@ private boolean isApplicationFinished(YarnApplicationState appState) { || appState == YarnApplicationState.KILLED; } - private int printAMContainerLogs(Configuration conf, - ContainerLogsRequest request, List amContainers, - String[] logFiles, LogCLIHelpers logCliHelper, String localDir) + private int printAMContainerLogs(Configuration conf, LogsCLIOptions request, + List amContainers, LogCLIHelpers logCliHelper) throws Exception { List amContainersList = null; - List requests = - new ArrayList(); + List requests = + new ArrayList(); boolean getAMContainerLists = false; String appId = request.getAppId().toString(); String errorMessage = ""; @@ -454,7 +403,7 @@ private int printAMContainerLogs(Configuration conf, if (amContainersList != null && !amContainersList.isEmpty()) { getAMContainerLists = true; for (JSONObject amContainer : amContainersList) { - ContainerLogsRequest amRequest = new ContainerLogsRequest(request); + LogsCLIOptions amRequest = new LogsCLIOptions(request); amRequest.setContainerId(amContainer.getString("containerId")); amRequest.setNodeHttpAddress( amContainer.getString("nodeHttpAddress")); @@ -470,7 +419,7 @@ private int printAMContainerLogs(Configuration conf, if (amContainersList != null && !amContainersList.isEmpty()) { getAMContainerLists = true; for (JSONObject amContainer : amContainersList) { - ContainerLogsRequest amRequest = new ContainerLogsRequest( + LogsCLIOptions amRequest = new LogsCLIOptions( request); amRequest.setContainerId(amContainer.getString("amContainerId")); requests.add(amRequest); @@ -489,34 +438,43 @@ private int printAMContainerLogs(Configuration conf, return -1; } + boolean foundAMLogs = false; if (amContainers.contains("ALL")) { - for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logFiles, - logCliHelper, localDir); + for (LogsCLIOptions amRequest : requests) { + int result = outputAMContainerLogs(amRequest, conf, logCliHelper); + if (result == 0) { + foundAMLogs = true; + } } System.out.println(); System.out.println("Specified ALL for -am option. " + "Printed logs for all am containers."); } else { for (String amContainer : amContainers) { + int result = -1; int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { - outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logFiles, logCliHelper, localDir); + result = outputAMContainerLogs(requests.get(requests.size() - 1), + conf, logCliHelper); } else { if (amContainerId <= requests.size()) { - outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logFiles, logCliHelper, localDir); + result = outputAMContainerLogs(requests.get(amContainerId - 1), + conf, logCliHelper); } } + if (result == 0) { + foundAMLogs = true; + } } } + if (!foundAMLogs) { + return -1; + } return 0; } - private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws Exception { + private int outputAMContainerLogs(LogsCLIOptions request, + Configuration conf, LogCLIHelpers logCliHelper) throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -531,55 +489,45 @@ private void outputAMContainerLogs(ContainerLogsRequest request, } catch (Exception ex) { System.err.println(ex); nodeId = null; + return -1; } } if (nodeId != null && !nodeId.isEmpty()) { - String[] requestedLogFilesList = null; - if(!fetchAllLogFiles(logFiles)) { - requestedLogFilesList = logFiles; - } - printContainerLogsForFinishedApplication(request, - requestedLogFilesList, logCliHelper, localDir); + return printContainerLogsForFinishedApplication(request, + logCliHelper); } } } else { if (nodeHttpAddress != null && containerId != null && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { - String[] requestedLogFiles = logFiles; - // fetch all the log files for the AM - if (fetchAllLogFiles(logFiles)) { - requestedLogFiles = - getContainerLogFiles(getConf(), containerId, nodeHttpAddress); - } printContainerLogsFromRunningApplication(conf, - request, requestedLogFiles, logCliHelper, localDir); + request, logCliHelper); + return 0; } } + return -1; } - private int showMetaInfo(ContainerLogsRequest request, + private int showMetaInfo(LogsCLIOptions request, LogCLIHelpers logCliHelper) throws IOException { if (!request.isAppFinished()) { System.err.println("The -show_meta_info command can be only used " + "with finished applications"); return -1; } else { - logCliHelper.printLogMetadata(request.getAppId(), - request.getContainerId(), request.getNodeId(), - request.getAppOwner(), System.out, System.err); + logCliHelper.printLogMetadata(request, System.out, System.err); return 0; } } - private int showNodeLists(ContainerLogsRequest request, + private int showNodeLists(LogsCLIOptions request, LogCLIHelpers logCliHelper) throws IOException { if (!request.isAppFinished()) { System.err.println("The -list_nodes command can be only used with " + "finished applications"); return -1; } else { - logCliHelper.printNodesList(request.getAppId(), request.getAppOwner(), - System.out, System.err); + logCliHelper.printNodesList(request, System.out, System.err); return 0; } } @@ -614,7 +562,7 @@ private Options createCommandOpts() { Option logFileOpt = new Option(CONTAINER_LOG_FILES, true, "Work with -am/-containerId and specify comma-separated value " + "to get specified container log files. Use \"ALL\" to fetch all the " - + "log files for the container."); + + "log files for the container. It also supports Java Regex."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); @@ -686,51 +634,36 @@ private Options createPrintOpts(Options commandOpts) { return amContainersList; } - private int fetchAMContainerLogs(ContainerLogsRequest request, - List amContainersList, String[] logFiles, - LogCLIHelpers logCliHelper, String localDir) throws Exception { - // if we do not specify the value for CONTAINER_LOG_FILES option, - // we will only output syslog - if (logFiles == null || logFiles.length == 0) { - logFiles = new String[] {"syslog"}; - } + private int fetchAMContainerLogs(LogsCLIOptions request, + List amContainersList, LogCLIHelpers logCliHelper) + throws Exception { // If the application is running, we will call the RM WebService // to get the AppAttempts which includes the nodeHttpAddress // and containerId for all the AM Containers. + // If the application is in the final state, we will call RM webservice + // to get all AppAttempts information first. If we get nothing, + // we will try to call AHS webservice to get related AppAttempts + // which includes nodeAddress for the AM Containers. // After that, we will call NodeManager webService to get the // related logs - if (!request.isAppFinished()) { - return printAMContainerLogs(getConf(), request, amContainersList, - logFiles, logCliHelper, localDir); - } else { - // If the application is in the final state, we will call RM webservice - // to get all AppAttempts information first. If we get nothing, - // we will try to call AHS webservice to get related AppAttempts - // which includes nodeAddress for the AM Containers. - // After that, we will use nodeAddress and containerId - // to get logs from HDFS directly. - if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, - YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { - return printAMContainerLogs(getConf(), request, amContainersList, - logFiles, logCliHelper, localDir); - } else { - ApplicationId appId = request.getAppId(); - String appOwner = request.getAppOwner(); - System.err.println("Can not get AMContainers logs for " - + "the application:" + appId + " with the appOwner:" + appOwner); - System.err.println("This application:" + appId + " has finished." - + " Please enable the application-history service or explicitly" - + " use 'yarn logs -applicationId " - + "-containerId --nodeAddress ' " - + "to get the container logs."); - return -1; - } - } + int result = printAMContainerLogs(getConf(), request, amContainersList, + logCliHelper); + if (result != 0) { + ApplicationId appId = request.getAppId(); + String appOwner = request.getAppOwner(); + System.err.println("Can not get AMContainers logs for " + + "the application:" + appId + " with the appOwner:" + appOwner); + System.err.println("This application:" + appId + " has finished." + + " Please enable the application-history service and explicitly" + + " use 'yarn logs -applicationId " + + "-containerId --nodeAddress ' " + + "to get the container logs."); + } + return result; } - private int fetchContainerLogs(ContainerLogsRequest request, - String[] logFiles, LogCLIHelpers logCliHelper, String localDir) - throws IOException { + private int fetchContainerLogs(LogsCLIOptions request, + LogCLIHelpers logCliHelper) throws IOException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -740,18 +673,7 @@ private int fetchContainerLogs(ContainerLogsRequest request, // if we provide the node address and the application is in the final // state, we could directly get logs from HDFS. if (nodeAddress != null && isAppFinished) { - // if user specified "ALL" as the logFiles param, pass null - // to logCliHelper so that it fetches all the logs - List logs; - if (logFiles == null) { - logs = null; - } else if (fetchAllLogFiles(logFiles)) { - logs = null; - } else { - logs = Arrays.asList(logFiles); - } - return logCliHelper.dumpAContainersLogsForALogType(appIdStr, - containerIdStr, nodeAddress, appOwner, logs, localDir); + return logCliHelper.dumpAContainersLogsForALogType(request); } String nodeHttpAddress = null; String nodeId = null; @@ -768,13 +690,8 @@ private int fetchContainerLogs(ContainerLogsRequest request, request.setNodeHttpAddress(nodeHttpAddress); } catch (IOException | YarnException ex) { if (isAppFinished) { - String[] requestedLogFiles = logFiles; - if(fetchAllLogFiles(logFiles)) { - requestedLogFiles = null; - } return printContainerLogsForFinishedApplicationWithoutNodeId( - appIdStr, containerIdStr, requestedLogFiles, logCliHelper, - appOwner, localDir); + request, logCliHelper); } else { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appIdStr @@ -790,31 +707,24 @@ private int fetchContainerLogs(ContainerLogsRequest request, // we will provide the NodeHttpAddress and get the container logs // by calling NodeManager webservice. if (!isAppFinished) { - if (logFiles == null || logFiles.length == 0) { - logFiles = new String[] {"syslog"}; - } printContainerLogsFromRunningApplication(getConf(), request, - logFiles, logCliHelper, localDir); + logCliHelper); } else { - String[] requestedLogFiles = logFiles; - if(fetchAllLogFiles(logFiles)) { - requestedLogFiles = null; - } // If the application is in the final state, we will directly // get the container logs from HDFS. - printContainerLogsForFinishedApplication(request, - requestedLogFiles, logCliHelper, localDir); + printContainerLogsForFinishedApplication(request, logCliHelper); } return resultCode; } - private int fetchApplicationLogs(ApplicationId appId, String appOwner, - LogCLIHelpers logCliHelper, String localDir) throws IOException { + private int fetchApplicationLogs(LogsCLIOptions request, + LogCLIHelpers logCliHelper) throws IOException { int resultCode = - logCliHelper.dumpAllContainersLogs(appId, appOwner, localDir); + logCliHelper.dumpAllContainersLogs(request); if (resultCode == -1) { System.err.println("Can not find the logs for the application: " - + appId + " with the appOwner: " + appOwner); + + request.getAppId() + " with the appOwner: " + + request.getAppOwner()); } return resultCode; } @@ -832,81 +742,4 @@ private String guessAppOwner(ApplicationReport appReport, } return appOwner; } - - private static class ContainerLogsRequest { - private ApplicationId appId; - private String containerId; - private String nodeId; - private String nodeHttpAddress; - private String appOwner; - private boolean appFinished; - - public ContainerLogsRequest(ContainerLogsRequest request) { - this.setAppId(request.getAppId()); - this.setAppFinished(request.isAppFinished()); - this.setAppOwner(request.getAppOwner()); - this.setNodeId(request.getNodeId()); - this.setNodeHttpAddress(request.getNodeHttpAddress()); - this.setContainerId(request.getContainerId()); - } - - public ContainerLogsRequest(ApplicationId applicationId, - boolean isAppFinished, String owner, - String address, String httpAddress, String container) { - this.setAppId(applicationId); - this.setAppFinished(isAppFinished); - this.setAppOwner(owner); - this.setNodeId(address); - this.setNodeHttpAddress(httpAddress); - this.setContainerId(container); - } - - public ApplicationId getAppId() { - return appId; - } - - public void setAppId(ApplicationId appId) { - this.appId = appId; - } - - public String getContainerId() { - return containerId; - } - - public void setContainerId(String containerId) { - this.containerId = containerId; - } - - public String getNodeId() { - return nodeId; - } - - public void setNodeId(String nodeAddress) { - this.nodeId = nodeAddress; - } - - public String getAppOwner() { - return appOwner; - } - - public void setAppOwner(String appOwner) { - this.appOwner = appOwner; - } - - public String getNodeHttpAddress() { - return nodeHttpAddress; - } - - public void setNodeHttpAddress(String nodeHttpAddress) { - this.nodeHttpAddress = nodeHttpAddress; - } - - public boolean isAppFinished() { - return appFinished; - } - - public void setAppFinished(boolean appFinished) { - this.appFinished = appFinished; - } - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index d649ce7..aad0e19 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -181,7 +181,8 @@ public void testHelpMessage() throws Exception { pw.println(" -logFiles Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); - pw.println(" all the log files for the container."); + pw.println(" all the log files for the container. It"); + pw.println(" also supports Java Regex."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.println(" -out Local directory for storing individual"); pw.println(" container logs. The container logs will"); @@ -288,6 +289,19 @@ public void testFetchApplictionLogs() throws Exception { "Hello container_0_0001_01_000003 in stdout!")); sysOutStream.reset(); + exitCode = cli.run(new String[] { "-applicationId", appId.toString(), + "-logFiles", ".*" }); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000001 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000002 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in syslog!")); + assertTrue(sysOutStream.toString().contains( + "Hello container_0_0001_01_000003 in stdout!")); + sysOutStream.reset(); + // uploaded two logs for container1. The first log is empty. // The second one is not empty. // We can still successfully read logs for container1. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 61b92dd..15f5f87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -771,7 +771,7 @@ public static int readContainerLogsForALogType( String fileType = valueStream.readUTF(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); - if (logType.contains(fileType)) { + if (isFileMatching(fileType, logType)) { out.print("LogType:"); out.println(fileType); if (logUploadedTime != -1) { @@ -808,6 +808,18 @@ public static int readContainerLogsForALogType( } } + private static boolean isFileMatching(String fileType, + List logTypes) { + for (String logType : logTypes) { + Pattern filterPattern = Pattern.compile(logType); + boolean match = filterPattern.matcher(fileType).find(); + if (match) { + return true; + } + } + return false; + } + @Private public static void readContainerMetaDataAndSkipData( DataInputStream valueStream, PrintStream out) throws IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 339df9d..6a2c794 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -26,6 +26,7 @@ import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -55,8 +56,15 @@ @VisibleForTesting public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { - return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner, - null, null); + LogsCLIOptions options = new LogsCLIOptions(); + options.setAppId(ConverterUtils.toApplicationId(appId)); + options.setContainerId(containerId); + options.setNodeId(nodeId); + options.setAppOwner(jobOwner); + List logs = new ArrayList(); + logs.add(".*"); + options.setLogTypes(logs); + return dumpAContainersLogsForALogType(options, false); } @Private @@ -108,21 +116,22 @@ public static String getOwnerForAppIdOrNull( @Private @VisibleForTesting - public int dumpAContainersLogsForALogType(String appId, String containerId, - String nodeId, String jobOwner, List logType, String localDir) + public int dumpAContainersLogsForALogType(LogsCLIOptions options) throws IOException { - return dumpAContainersLogsForALogType(appId, containerId, nodeId, - jobOwner, logType, true, localDir); + return dumpAContainersLogsForALogType(options, true); } @Private @VisibleForTesting - public int dumpAContainersLogsForALogType(String appId, String containerId, - String nodeId, String jobOwner, List logType, - boolean outputFailure, String localDir) throws IOException { - ApplicationId applicationId = ConverterUtils.toApplicationId(appId); + public int dumpAContainersLogsForALogType(LogsCLIOptions options, + boolean outputFailure) throws IOException { + ApplicationId applicationId = options.getAppId(); + String containerId = options.getContainerId(); + String nodeId = options.getNodeId(); + String localDir = options.getOutputLocalDir(); + List logType = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( - applicationId, jobOwner); + applicationId, options.getAppOwner()); if (nodeFiles == null) { return -1; } @@ -147,16 +156,9 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); - if (logType == null) { - if (dumpAContainerLogs(containerId, reader, out, - thisNodeFile.getModificationTime()) > -1) { - foundContainerLogs = true; - } - } else { - if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { - foundContainerLogs = true; - } + if (dumpAContainerLogsForALogType(containerId, reader, out, + thisNodeFile.getModificationTime(), logType) > -1) { + foundContainerLogs = true; } } finally { if (reader != null) { @@ -176,12 +178,14 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, } @Private - public int dumpAContainersLogsForALogTypeWithoutNodeId(String appId, - String containerId, String jobOwner, List logType, - String localDir) throws IOException { - ApplicationId applicationId = ConverterUtils.toApplicationId(appId); + public int dumpAContainersLogsForALogTypeWithoutNodeId( + LogsCLIOptions options) throws IOException { + ApplicationId applicationId = options.getAppId(); + String containerId = options.getContainerId(); + String localDir = options.getOutputLocalDir(); + List logType = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( - applicationId, jobOwner); + applicationId, options.getAppOwner()); if (nodeFiles == null) { return -1; } @@ -206,16 +210,9 @@ public int dumpAContainersLogsForALogTypeWithoutNodeId(String appId, containerId); out.println(containerId); out.println(StringUtils.repeat("=", containerId.length())); - if (logType == null) { - if (dumpAContainerLogs(containerId, reader, out, - thisNodeFile.getModificationTime()) > -1) { - foundContainerLogs = true; - } - } else { - if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { - foundContainerLogs = true; - } + if (dumpAContainerLogsForALogType(containerId, reader, out, + thisNodeFile.getModificationTime(), logType) > -1) { + foundContainerLogs = true; } } finally { if (reader != null) { @@ -303,10 +300,12 @@ public int dumpAContainerLogsForALogType(String containerIdStr, } @Private - public int dumpAllContainersLogs(ApplicationId appId, String appOwner, - String localDir) throws IOException { + public int dumpAllContainersLogs(LogsCLIOptions options) throws IOException { + ApplicationId appId = options.getAppId(); + String localDir = options.getOutputLocalDir(); + List logTypes = options.getLogTypes(); RemoteIterator nodeFiles = getRemoteNodeFileDir( - appId, appOwner); + appId, options.getAppOwner()); if (nodeFiles == null) { return -1; } @@ -341,9 +340,17 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { - LogReader.readAContainerLogsForALogType(valueStream, out, - thisNodeFile.getModificationTime()); - foundAnyLogs = true; + if (logTypes == null) { + LogReader.readAContainerLogsForALogType(valueStream, out, + thisNodeFile.getModificationTime()); + foundAnyLogs = true; + } else { + int result = LogReader.readContainerLogsForALogType(valueStream, + out, thisNodeFile.getModificationTime(), logTypes); + if (result == 0) { + foundAnyLogs = true; + } + } } catch (EOFException eof) { break; } @@ -362,17 +369,20 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, } } if (!foundAnyLogs) { - emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString()); + emptyLogDir(getRemoteAppLogDir(appId, options.getAppOwner()).toString()); return -1; } return 0; } @Private - public void printLogMetadata(ApplicationId appId, - String containerIdStr, String nodeId, String appOwner, + public void printLogMetadata(LogsCLIOptions options, PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); + String containerIdStr = options.getContainerId(); + String nodeId = options.getNodeId(); + String appOwner = options.getAppOwner(); boolean getAllContainers = (containerIdStr == null); String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); @@ -443,10 +453,11 @@ public void printLogMetadata(ApplicationId appId, } @Private - public void printNodesList(ApplicationId appId, String appOwner, + public void printNodesList(LogsCLIOptions options, PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); RemoteIterator nodeFiles = getRemoteNodeFileDir( - appId, appOwner); + appId, options.getAppOwner()); if (nodeFiles == null) { return; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java new file mode 100644 index 0000000..3f9a4f0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import java.util.List; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +public class LogsCLIOptions { + private ApplicationId appId; + private String containerId; + private String nodeId; + private String nodeHttpAddress; + private String appOwner; + private boolean appFinished; + private String outputLocalDir; + private List LogTypes; + + public LogsCLIOptions() {} + + public LogsCLIOptions(LogsCLIOptions request) { + this.setAppId(request.getAppId()); + this.setAppFinished(request.isAppFinished()); + this.setAppOwner(request.getAppOwner()); + this.setNodeId(request.getNodeId()); + this.setNodeHttpAddress(request.getNodeHttpAddress()); + this.setContainerId(request.getContainerId()); + this.setOutputLocalDir(request.getOutputLocalDir()); + this.setLogTypes(request.getLogTypes()); + } + + public LogsCLIOptions(ApplicationId applicationId, + boolean isAppFinished, String owner, + String address, String httpAddress, String container, String localDir, + List logs) { + this.setAppId(applicationId); + this.setAppFinished(isAppFinished); + this.setAppOwner(owner); + this.setNodeId(address); + this.setNodeHttpAddress(httpAddress); + this.setContainerId(container); + this.setOutputLocalDir(localDir); + this.setLogTypes(logs); + } + + public ApplicationId getAppId() { + return appId; + } + + public void setAppId(ApplicationId appId) { + this.appId = appId; + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeAddress) { + this.nodeId = nodeAddress; + } + + public String getAppOwner() { + return appOwner; + } + + public void setAppOwner(String appOwner) { + this.appOwner = appOwner; + } + + public String getNodeHttpAddress() { + return nodeHttpAddress; + } + + public void setNodeHttpAddress(String nodeHttpAddress) { + this.nodeHttpAddress = nodeHttpAddress; + } + + public boolean isAppFinished() { + return appFinished; + } + + public void setAppFinished(boolean appFinished) { + this.appFinished = appFinished; + } + + public String getOutputLocalDir() { + return outputLocalDir; + } + + public void setOutputLocalDir(String outputLocalDir) { + this.outputLocalDir = outputLocalDir; + } + + public List getLogTypes() { + return LogTypes; + } + + public void setLogTypes(List logTypes) { + LogTypes = logTypes; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index deae894..502cebb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -23,9 +23,11 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; - +import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; @@ -313,7 +315,7 @@ private Response sendStreamOutputResponse(ApplicationId appId, if (downloadFile) { response.header("Content-Type", "application/octet-stream"); response.header("Content-Disposition", "attachment; filename=" - + fileName); + + containerIdStr); } return response.build(); } @@ -381,7 +383,7 @@ public void write(OutputStream os) throws IOException, String fileType = valueStream.readUTF(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); - if (fileType.equalsIgnoreCase(logFile)) { + if (isFileMatched(logFile, fileType)) { StringBuilder sb = new StringBuilder(); sb.append("LogType:"); sb.append(fileType + "\n"); @@ -435,4 +437,9 @@ public void write(OutputStream os) throws IOException, }; return stream; } + + private boolean isFileMatched(String filePattern, String fileName) { + Pattern filterPattern = Pattern.compile(filePattern); + return filterPattern.matcher(fileName).find(); + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index f985fe4..e87b733 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -501,6 +501,7 @@ public void testSingleContainer() throws Exception { @Test(timeout = 10000) public void testContainerLogsForFinishedApps() throws Exception { String fileName = "syslog"; + String fileName2 = "stderr"; String user = "user1"; UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user1"); NodeId nodeId = NodeId.newInstance("test host", 100); @@ -522,6 +523,7 @@ public void testContainerLogsForFinishedApps() throws Exception { } assertTrue(fs.mkdirs(appLogsDir)); + List fileContexts = new ArrayList(); // create container logs in local log file dir // create two container log files. We can get containerInfo // for container1 from AHS, but can not get such info for @@ -529,11 +531,22 @@ public void testContainerLogsForFinishedApps() throws Exception { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); - ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); - createContainerLogInLocalDir(appLogsDir, containerId1, fs, fileName, + LogFileContext fileContext1 = new LogFileContext(fileName, ("Hello." + containerId1)); - createContainerLogInLocalDir(appLogsDir, containerId100, fs, fileName, + LogFileContext fileContext2 = new LogFileContext(fileName2, + ("Error." + containerId1)); + fileContexts.add(fileContext1); + fileContexts.add(fileContext2); + createContainerLogInLocalDir(appLogsDir, containerId1, fs, fileContexts); + ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); + fileContexts.clear(); + fileContext1 = new LogFileContext(fileName, ("Hello." + containerId100)); + fileContext2 = new LogFileContext(fileName2, + ("Error." + containerId100)); + fileContexts.add(fileContext1); + fileContexts.add(fileContext2); + createContainerLogInLocalDir(appLogsDir, containerId100, fs, fileContexts); // upload container logs to remote log dir Path path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) + @@ -559,6 +572,17 @@ public void testContainerLogsForFinishedApps() throws Exception { String responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1)); + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1.toString()).path(".*") + .queryParam("user.name", user) + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains("Hello." + containerId1)); + assertTrue(responseText.contains("Error." + containerId1)); + // test whether we can find container log from remote diretory if // the containerInfo for this container could not be fetched from AHS. r = resource(); @@ -571,6 +595,17 @@ public void testContainerLogsForFinishedApps() throws Exception { responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId100)); + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId100.toString()).path(".*") + .queryParam("user.name", user) + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains("Hello." + containerId100)); + assertTrue(responseText.contains("Error." + containerId100)); + // create an application which can not be found from AHS ApplicationId appId100 = ApplicationId.newInstance(0, 100); appLogsDir = new Path(rootLogDirPath, appId100.toString()); @@ -582,8 +617,15 @@ public void testContainerLogsForFinishedApps() throws Exception { ApplicationAttemptId.newInstance(appId100, 1); ContainerId containerId1ForApp100 = ContainerId .newContainerId(appAttemptId100, 1); + fileContexts.clear(); + fileContext1 = new LogFileContext(fileName, + ("Hello." + containerId1ForApp100)); + fileContext2 = new LogFileContext(fileName2, + ("Error." + containerId1ForApp100)); + fileContexts.add(fileContext1); + fileContexts.add(fileContext2); createContainerLogInLocalDir(appLogsDir, containerId1ForApp100, fs, - fileName, ("Hello." + containerId1ForApp100)); + fileContexts); path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) + user + "/logs/" + appId100.toString()); if (fs.exists(path)) { @@ -601,20 +643,34 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1ForApp100)); + + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1ForApp100.toString()).path(".*") + .queryParam("user.name", user) + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains("Hello." + containerId1ForApp100)); + assertTrue(responseText.contains("Error." + containerId1ForApp100)); } private static void createContainerLogInLocalDir(Path appLogsDir, - ContainerId containerId, FileSystem fs, String fileName, String content) + ContainerId containerId, FileSystem fs, List fileContexts) throws Exception { Path containerLogsDir = new Path(appLogsDir, containerId.toString()); if (fs.exists(containerLogsDir)) { fs.delete(containerLogsDir, true); } assertTrue(fs.mkdirs(containerLogsDir)); - Writer writer = - new FileWriter(new File(containerLogsDir.toString(), fileName)); - writer.write(content); - writer.close(); + for (LogFileContext fileContext : fileContexts) { + Writer writer = + new FileWriter(new File(containerLogsDir.toString(), + fileContext.getFileName())); + writer.write(fileContext.getFileContent()); + writer.close(); + } } private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, @@ -670,4 +726,25 @@ private static String getRedirectURL(String url) { } return redirectUrl; } + + private class LogFileContext { + private String fileName; + private String fileContent; + public LogFileContext(String name, String content) { + this.setFileName(name); + this.setFileContent(content); + } + public String getFileName() { + return fileName; + } + public void setFileName(String fileName) { + this.fileName = fileName; + } + public String getFileContent() { + return fileContent; + } + public void setFileContent(String fileContent) { + this.fileContent = fileContent; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 57e729c..0a675f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -21,8 +21,10 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; import java.util.Map.Entry; - +import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; @@ -37,7 +39,7 @@ import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; - +import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -215,39 +217,48 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context @Produces({ MediaType.TEXT_PLAIN }) @Public @Unstable - public Response getLogs(@PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("download") String download) { - ContainerId containerId; + public Response getLogs(@javax.ws.rs.core.Context + HttpServletRequest hsr, @PathParam("containerid") + final String containerIdStr, @PathParam("filename") String filename, + @QueryParam("download") String download) throws IOException { + ContainerId tempId; try { - containerId = ConverterUtils.toContainerId(containerIdStr); + tempId = ConverterUtils.toContainerId(containerIdStr); } catch (IllegalArgumentException ex) { return Response.status(Status.BAD_REQUEST).build(); } - - File logFile = null; - try { - logFile = ContainerLogsUtils.getContainerLogFile( - containerId, filename, request.getRemoteUser(), nmContext); - } catch (NotFoundException ex) { - return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build(); - } catch (YarnException ex) { - return Response.serverError().entity(ex.getMessage()).build(); + final ContainerId containerId = tempId; + ContainerInfo containerInfo = getNodeContainer(hsr, containerIdStr); + final List matchedFiles = findMatchingFiles(filename, + containerInfo.getContainerLogFiles()); + if (matchedFiles.isEmpty()) { + String errorMessage = "Can not find any log files matching " + + "the pattern: " + filename + " on the local disk."; + return Response.status(Status.NOT_FOUND).entity(errorMessage).build(); } + boolean downloadFile = parseBooleanParam(download); - try { - final FileInputStream fis = ContainerLogsUtils.openLogFileForRead( - containerIdStr, logFile, nmContext); - - StreamingOutput stream = new StreamingOutput() { + StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, WebApplicationException { int bufferSize = 65536; byte[] buf = new byte[bufferSize]; - int len; - while ((len = fis.read(buf, 0, bufferSize)) > 0) { - os.write(buf, 0, len); + File logFile = null; + for (String file : matchedFiles) { + try { + logFile = ContainerLogsUtils.getContainerLogFile( + containerId, file, request.getRemoteUser(), nmContext); + } catch (YarnException | NotFoundException ex) { + continue; + } + FileInputStream fis = ContainerLogsUtils.openLogFileForRead( + containerIdStr, logFile, nmContext); + int len; + os.write(("\n" + file + "\n").getBytes()); + while ((len = fis.read(buf, 0, bufferSize)) > 0) { + os.write(buf, 0, len); + } } os.flush(); } @@ -256,12 +267,21 @@ public void write(OutputStream os) throws IOException, if (downloadFile) { resp.header("Content-Type", "application/octet-stream"); resp.header("Content-Disposition", "attachment; filename=" - + logFile.getName()); + + containerIdStr); } return resp.build(); - } catch (IOException ex) { - return Response.serverError().entity(ex.getMessage()).build(); + } + + private List findMatchingFiles(String filename, + List containerLogFiles) { + Pattern filterPattern = Pattern.compile(filename); + List files = new ArrayList(); + for (String logFile : containerLogFiles) { + if (filterPattern.matcher(logFile).find()) { + files.add(logFile); + } } + return files; } private boolean parseBooleanParam(String param) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index ce1b309..860dda6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -23,10 +23,11 @@ import static org.junit.Assert.fail; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.StringReader; - +import java.io.Writer; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -313,18 +314,19 @@ public void testSingleNodesXML() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, nodes.getLength()); verifyNodesXML(nodes); } - + @Test public void testContainerLogs() throws IOException { WebResource r = resource(); - final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0); - final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0) - .toString(); + final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 1); + final String containerIdStr = containerId.toString(); final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); final ApplicationId appId = appAttemptId.getApplicationId(); final String appIdStr = appId.toString(); final String filename = "logfile1"; final String logMessage = "log message\n"; + final String filename2 = "stderr"; + final String errorMessage = "error message\n"; nmContext.getApplications().put(appId, new ApplicationImpl(null, "user", appId, null, nmContext)); @@ -333,7 +335,7 @@ public void testContainerLogs() throws IOException { container.setState(ContainerState.RUNNING); nmContext.getContainers().put(containerId, container); - // write out log file + // write out log file for "logfile1" Path path = dirsHandler.getLogPathForWrite( ContainerLaunch.getRelativeContainerLogDir( appIdStr, containerIdStr) + "/" + filename, false); @@ -345,19 +347,37 @@ public void testContainerLogs() throws IOException { pw.print(logMessage); pw.close(); + // write out log file for "stderr" + path = dirsHandler.getLogPathForWrite( + ContainerLaunch.getRelativeContainerLogDir( + appIdStr, containerIdStr) + "/" + filename2, false); + + logFile = new File(path.toUri().getPath()); + logFile.deleteOnExit(); + pw = new PrintWriter(logFile); + pw.print(errorMessage); + pw.close(); + // ask for it ClientResponse response = r.path("ws").path("v1").path("node") .path("containerlogs").path(containerIdStr).path(filename) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); String responseText = response.getEntity(String.class); - assertEquals(logMessage, responseText); + Assert.assertTrue(responseText.contains(logMessage)); + + response = r.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(".*") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + Assert.assertTrue(responseText.contains(logMessage)); + Assert.assertTrue(responseText.contains(errorMessage)); // ask and download it response = r.path("ws").path("v1").path("node").path("containerlogs") .path(containerIdStr).path(filename).queryParam("download", "true") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); - assertEquals(logMessage, responseText); + Assert.assertTrue(responseText.contains(logMessage)); assertEquals(200, response.getStatus()); assertEquals("application/octet-stream", response.getType().toString()); @@ -367,7 +387,8 @@ public void testContainerLogs() throws IOException { .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); responseText = response.getEntity(String.class); - assertTrue(responseText.contains("Cannot find this log on the local disk.")); + assertTrue(responseText.contains("Can not find any log files " + + "matching the pattern")); // After container is completed, it is removed from nmContext nmContext.getContainers().remove(containerId); @@ -376,8 +397,8 @@ public void testContainerLogs() throws IOException { r.path("ws").path("v1").path("node").path("containerlogs") .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN) .get(ClientResponse.class); - responseText = response.getEntity(String.class); - assertEquals(logMessage, responseText); + assertEquals(Status.INTERNAL_SERVER_ERROR.getStatusCode(), + response.getStatus()); } public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {