diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..7f161ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -78,6 +78,7 @@ private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; + private static final String LS_OPTION = "ls"; public static final String HELP_CMD = "help"; @Override @@ -117,6 +118,12 @@ public int run(String[] args) throws Exception { logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(LS_OPTION, false, "List Container log meta," + + "including log file names, the size of the log files. Specify " + + "--containerId to get log meta for the specific container. " + + "Or specify --nodeAddress to get all container log meta for " + + "the specific nodemanager. Currently, this option can only be used " + + "when this applicaiton is finished."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); @@ -131,6 +138,7 @@ public int run(String[] args) throws Exception { printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); + printOpts.addOption(opts.getOption(LS_OPTION)); if (args.length < 1) { printHelpMessage(printOpts); @@ -146,6 +154,7 @@ public int run(String[] args) throws Exception { String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; + boolean ls = false; String[] logFiles = null; List amContainersList = new ArrayList(); try { @@ -155,6 +164,7 @@ public int run(String[] args) throws Exception { nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); + ls = commandLine.hasOption(LS_OPTION); if (getAMContainerLogs) { String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); for (String am : amContainers) { @@ -226,6 +236,17 @@ public int run(String[] args) throws Exception { + " Attempting to fetch logs directly from the filesystem."); } + if (ls) { + if (! isApplicationFinished(appState)) { + System.out.println("The LS command can be only used " + + "when the Application is finished"); + return -1; + } else { + logCliHelper.listContainerLogInfo(appId, containerIdStr, + nodeAddress, appOwner, System.out); + return 0; + } + } // To get am logs if (getAMContainerLogs) { // if we do not specify the value for CONTAINER_LOG_FILES option, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aec7cae..b1a2b4c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -170,6 +170,14 @@ public void testHelpMessage() throws Exception { pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); pw.println(" all the log files for the container."); + pw.println(" -ls List Container log meta,including log"); + pw.println(" file names, the size of the log files."); + pw.println(" Specify --containerId to get log meta for"); + pw.println(" the specific container. Or specify"); + pw.println(" --nodeAddress to get all container log"); + pw.println(" meta for the specific nodemanager."); + pw.println(" Currently, this option can only be used"); + pw.println(" when this applicaiton is finished."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.close(); String appReportStr = baos.toString("UTF-8"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index c9453b3..84b6b9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -808,6 +808,26 @@ public static int readContainerLogsForALogType( } } + public static int listContainerLogs(DataInputStream valueStream, + PrintStream out) throws IOException { + + String fileType = valueStream.readUTF(); + String fileLengthStr = valueStream.readUTF(); + long fileLength = Long.parseLong(fileLengthStr); + out.print("LogType:"); + out.println(fileType); + out.print("LogLength:"); + out.println(fileLengthStr); + + long totalSkipped = 0; + long currSkipped = 0; + while (currSkipped != -1 && totalSkipped < fileLength) { + currSkipped = valueStream.skip(fileLength - totalSkipped); + totalSkipped += currSkipped; + } + return 0; + } + public void close() { IOUtils.cleanup(LOG, scanner, reader, fsDataIStream); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..ccef597 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -263,6 +263,84 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, return 0; } + @Private + public void listContainerLogInfo(ApplicationId appId, String containerIdStr, + String nodeId, String appOwner, PrintStream out) throws IOException { + boolean getAllContainers = (containerIdStr == null); + Path remoteRootLogDir = new Path(getConf().get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String user = appOwner; + String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); + // TODO Change this to get a list of files from the LAS. + Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( + remoteRootLogDir, appId, user, logDirSuffix); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator nodeFiles; + try { + Path qualifiedLogDir = + FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); + nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), + getConf()).listStatus(remoteAppLogDir); + } catch (FileNotFoundException fnf) { + logDirNotExist(remoteAppLogDir.toString()); + return; + } + boolean foundAnyLogs = false; + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + if (getAllContainers || (key.toString().equals(containerIdStr))) { + String containerString = + "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName(); + out.println(containerString); + out.println("Log Upload Time:" + thisNodeFile.getModificationTime()); + out.println(StringUtils.repeat("=", containerString.length())); + while (true) { + try { + LogReader.listContainerLogs(valueStream, out); + } catch (EOFException eof) { + break; + } + } + foundAnyLogs = true; + } + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + if (!foundAnyLogs) { + if (containerIdStr != null && nodeId != null) { + out.println("Can not find log meta for container: " + containerIdStr + + " on " + nodeId); + } else if (nodeId != null) { + out.println("Can not find log meta for any containers on " + nodeId); + } else if (containerIdStr != null) { + out.println("Can not find log meta for container: " + containerIdStr); + } + } + } + @Override public void setConf(Configuration conf) { this.conf = conf;