diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..e5e2279 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.cli; import java.io.IOException; +import java.io.PrintStream; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; @@ -78,6 +79,7 @@ private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; + private static final String OUT_OPTION = "out"; public static final String HELP_CMD = "help"; @Override @@ -118,11 +120,15 @@ public int run(String[] args) throws Exception { logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(OUT_OPTION, true, "Local File Path. Currently " + + "only support when the app is finished."); + opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers"); + opts.getOption(OUT_OPTION).setArgName("Local File Path"); Options printOpts = new Options(); printOpts.addOption(opts.getOption(HELP_CMD)); @@ -131,6 +137,7 @@ public int run(String[] args) throws Exception { printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); + printOpts.addOption(opts.getOption(OUT_OPTION)); if (args.length < 1) { printHelpMessage(printOpts); @@ -148,12 +155,14 @@ public int run(String[] args) throws Exception { boolean getAMContainerLogs = false; String[] logFiles = null; List amContainersList = new ArrayList(); + String localPath = null; try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); + localPath = commandLine.getOptionValue(OUT_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); if (getAMContainerLogs) { String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); @@ -226,6 +235,13 @@ public int run(String[] args) throws Exception { + " Attempting to fetch logs directly from the filesystem."); } + if (localPath != null) { + if (!isApplicationFinished(appState)) { + System.out.println("The option: --" + OUT_OPTION + + " can only be used when the application is finished"); + return -1; + } + } // To get am logs if (getAMContainerLogs) { // if we do not specify the value for CONTAINER_LOG_FILES option, @@ -241,7 +257,7 @@ public int run(String[] args) throws Exception { if (appState == YarnApplicationState.ACCEPTED || appState == YarnApplicationState.RUNNING) { return printAMContainerLogs(getConf(), appIdStr, amContainersList, - logFiles, logCliHelper, appOwner, false); + logFiles, logCliHelper, appOwner, false, null); } else { // If the application is in the final state, we will call RM webservice // to get all AppAttempts information first. If we get nothing, @@ -252,7 +268,7 @@ public int run(String[] args) throws Exception { if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { return printAMContainerLogs(getConf(), appIdStr, amContainersList, - logFiles, logCliHelper, appOwner, true); + logFiles, logCliHelper, appOwner, true, localPath); } else { System.out .println( @@ -281,8 +297,18 @@ public int run(String[] args) throws Exception { } else { logs = Arrays.asList(logFiles); } - return logCliHelper.dumpAContainersLogsForALogType(appIdStr, - containerIdStr, nodeAddress, appOwner, logs); + if (localPath != null) { + PrintStream p = new PrintStream(localPath, "UTF-8"); + try { + return logCliHelper.dumpAContainersLogsForALogType(appIdStr, + containerIdStr, nodeAddress, appOwner, logs, p); + } finally { + p.close(); + } + } else { + return logCliHelper.dumpAContainersLogsForALogType(appIdStr, + containerIdStr, nodeAddress, appOwner, logs, System.out); + } } try { // If the nodeAddress is not provided, we will try to get @@ -311,7 +337,7 @@ public int run(String[] args) throws Exception { // If the application is in the final state, we will directly // get the container logs from HDFS. printContainerLogsForFinishedApplication(appIdStr, containerIdStr, - nodeId, requestedLogFiles, logCliHelper, appOwner); + nodeId, requestedLogFiles, logCliHelper, appOwner, localPath); } return resultCode; } catch (IOException | YarnException ex) { @@ -328,8 +354,18 @@ public int run(String[] args) throws Exception { } } else { if (nodeAddress == null) { - resultCode = - logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); + if (localPath != null) { + PrintStream p = new PrintStream(localPath, "UTF-8"); + try { + resultCode = + logCliHelper.dumpAllContainersLogs(appId, appOwner, p); + } finally { + p.close(); + } + } else { + resultCode = + logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); + } } else { System.out.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -506,17 +542,28 @@ private void printContainerLogsFromRunningApplication(Configuration conf, } // for the case, we have already uploaded partial logs in HDFS logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId, - appOwner, Arrays.asList(requestedLogFiles)); + appOwner, Arrays.asList(requestedLogFiles), System.out); } private void printContainerLogsForFinishedApplication(String appId, String containerId, String nodeAddress, String[] logFiles, - LogCLIHelpers logCliHelper, String appOwner) throws IOException { + LogCLIHelpers logCliHelper, String appOwner, String localPath) + throws IOException { String containerString = "\n\nContainer: " + containerId; - System.out.println(containerString); - System.out.println(StringUtils.repeat("=", containerString.length())); - logCliHelper.dumpAContainersLogsForALogType(appId, containerId, - nodeAddress, appOwner, logFiles != null ? Arrays.asList(logFiles) : null); + PrintStream p = System.out; + if (localPath != null) { + p = new PrintStream(localPath, "UTF-8"); + } + try { + p.println(containerString); + p.println(StringUtils.repeat("=", containerString.length())); + logCliHelper.dumpAContainersLogsForALogType(appId, containerId, + nodeAddress, appOwner, + logFiles != null ? Arrays.asList(logFiles) : null, + p); + } finally { + p.close(); + } } private ContainerReport getContainerReport(String containerIdStr) @@ -538,7 +585,8 @@ private boolean isApplicationFinished(YarnApplicationState appState) { private int printAMContainerLogs(Configuration conf, String appId, List amContainers, String[] logFiles, LogCLIHelpers logCliHelper, - String appOwner, boolean applicationFinished) throws Exception { + String appOwner, boolean applicationFinished, String localPath) + throws Exception { List amContainersList = null; List requests = new ArrayList(); boolean getAMContainerLists = false; @@ -584,7 +632,7 @@ private int printAMContainerLogs(Configuration conf, String appId, if (amContainers.contains("ALL")) { for (AMLogsRequest request : requests) { outputAMContainerLogs(request, conf, appId, logFiles, logCliHelper, - appOwner); + appOwner, localPath); } System.out.println(); System.out.println("Specified ALL for -am option. " @@ -594,11 +642,11 @@ private int printAMContainerLogs(Configuration conf, String appId, int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { outputAMContainerLogs(requests.get(requests.size() - 1), conf, appId, - logFiles, logCliHelper, appOwner); + logFiles, logCliHelper, appOwner, localPath); } else { if (amContainerId <= requests.size()) { outputAMContainerLogs(requests.get(amContainerId - 1), conf, appId, - logFiles, logCliHelper, appOwner); + logFiles, logCliHelper, appOwner, localPath); } } } @@ -608,7 +656,7 @@ private int printAMContainerLogs(Configuration conf, String appId, private void outputAMContainerLogs(AMLogsRequest request, Configuration conf, String appId, String[] logFiles, LogCLIHelpers logCliHelper, - String appOwner) throws Exception { + String appOwner, String localPath) throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getAmContainerId(); String nodeId = request.getNodeId(); @@ -630,7 +678,7 @@ private void outputAMContainerLogs(AMLogsRequest request, Configuration conf, requestedLogFilesList = logFiles; } printContainerLogsForFinishedApplication(appId, containerId, nodeId, - requestedLogFilesList, logCliHelper, appOwner); + requestedLogFilesList, logCliHelper, appOwner, localPath); } } } else { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aec7cae..164fcf2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -171,6 +171,8 @@ public void testHelpMessage() throws Exception { pw.println(" container log files. Use \"ALL\" to fetch"); pw.println(" all the log files for the container."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); + pw.println(" -out Local File Path. Currently only support"); + pw.println(" when the app is finished."); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..3d5a920 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -51,13 +51,14 @@ public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner, - null); + null, System.out); } @Private @VisibleForTesting public int dumpAContainersLogsForALogType(String appId, String containerId, - String nodeId, String jobOwner, List logType) throws IOException { + String nodeId, String jobOwner, List logType, PrintStream out) + throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); @@ -96,12 +97,12 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); if (logType == null) { - if (dumpAContainerLogs(containerId, reader, System.out, + if (dumpAContainerLogs(containerId, reader, out, thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; } } else { - if (dumpAContainerLogsForALogType(containerId, reader, System.out, + if (dumpAContainerLogsForALogType(containerId, reader, out, thisNodeFile.getModificationTime(), logType) > -1) { foundContainerLogs = true; }