diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..00df84e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -78,6 +78,8 @@ private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; + private static final String SHOW_META_INFO = "show_meta_info"; + private static final String LIST_NODES_OPTION = "list_nodes"; public static final String HELP_CMD = "help"; @Override @@ -117,6 +119,16 @@ public int run(String[] args) throws Exception { logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(SHOW_META_INFO, false, "Show the log metadata, " + + "including log-file names, the size of the log files. " + + "You can combine this with --containerId to get log metadata for " + + "the specific container, or with --nodeAddress to get log metadata " + + "for all the containers on the specific NodeManager. " + + "Currently, this option can only be used for finished " + + "applications."); + opts.addOption(LIST_NODES_OPTION, false, + "Show the list of nodes that successfully aggregated logs. " + + "This option can only be used with finished applications."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); @@ -131,6 +143,8 @@ public int run(String[] args) throws Exception { printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); + printOpts.addOption(opts.getOption(SHOW_META_INFO)); + printOpts.addOption(opts.getOption(LIST_NODES_OPTION)); if (args.length < 1) { printHelpMessage(printOpts); @@ -146,6 +160,8 @@ public int run(String[] args) throws Exception { String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; + boolean showMetaInfo = false; + boolean nodesList = false; String[] logFiles = null; List amContainersList = new ArrayList(); try { @@ -155,6 +171,8 @@ public int run(String[] args) throws Exception { nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); + showMetaInfo = commandLine.hasOption(SHOW_META_INFO); + nodesList = commandLine.hasOption(LIST_NODES_OPTION); if (getAMContainerLogs) { String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); for (String am : amContainers) { @@ -212,18 +230,43 @@ public int run(String[] args) throws Exception { appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); } + boolean appStateObtainedSuccessfully = true; YarnApplicationState appState = YarnApplicationState.NEW; try { appState = getApplicationState(appId); if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) { - System.out.println("Logs are not avaiable right now."); + System.err.println("Logs are not avaiable right now."); return -1; } } catch (IOException | YarnException e) { System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem."); + appStateObtainedSuccessfully = false; + } + + if (showMetaInfo) { + if (!isApplicationFinished(appState) && appStateObtainedSuccessfully) { + System.err.println("The -show_meta_info command can be only used " + + "with finished applications"); + return -1; + } else { + logCliHelper.printLogMetadata(appId, containerIdStr, nodeAddress, + appOwner, System.out, System.err); + return 0; + } + } + + if (nodesList) { + if (!isApplicationFinished(appState) && appStateObtainedSuccessfully) { + System.err.println("The -list-nodes command can be only used with " + + "finished applications"); + return -1; + } else { + logCliHelper.printNodesList(appId, appOwner, System.out, System.err); + return 0; + } } // To get am logs @@ -254,10 +297,9 @@ public int run(String[] args) throws Exception { return printAMContainerLogs(getConf(), appIdStr, amContainersList, logFiles, logCliHelper, appOwner, true); } else { - System.out - .println( - "Can not get AMContainers logs for the application:" + appId); - System.out.println("This application:" + appId + " is finished." + System.err.println("Can not get AMContainers logs " + + "for the application:" + appId); + System.err.println("This application:" + appId + " is finished." + " Please enable the application history service. Or Using " + "yarn logs -applicationId -containerId " + "--nodeAddress to get the container logs"); @@ -319,9 +361,9 @@ public int run(String[] args) throws Exception { + containerIdStr + "for the application:" + appId); if (!getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { - System.out.println("Please enable the application history service. Or "); + System.err.println("Please enable the application history service. Or "); } - System.out.println("Using " + System.err.println("Using " + "yarn logs -applicationId -containerId " + "--nodeAddress to get the container logs"); return -1; @@ -331,7 +373,7 @@ public int run(String[] args) throws Exception { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else { - System.out.println("Should at least provide ContainerId!"); + System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); resultCode = -1; } @@ -455,14 +497,14 @@ private boolean fetchAllLogFiles(String[] logFiles) { logFiles.add(elements.item(i).getTextContent()); } } catch (Exception e) { - System.out.println("Unable to parse xml from webservice. Error:"); - System.out.println(e.getMessage()); + System.err.println("Unable to parse xml from webservice. Error:"); + System.err.println(e.getMessage()); throw new IOException(e); } } } catch (ClientHandlerException | UniformInterfaceException ex) { - System.out.println("Unable to fetch log files list"); + System.err.println("Unable to fetch log files list"); throw new IOException(ex); } return logFiles.toArray(new String[0]); @@ -499,7 +541,7 @@ private void printContainerLogsFromRunningApplication(Configuration conf, System.out.println(response.getEntity(String.class)); System.out.println("End of LogType:" + logFile); } catch (ClientHandlerException | UniformInterfaceException ex) { - System.out.println("Can not find the log file:" + logFile + System.err.println("Can not find the log file:" + logFile + " for the container:" + containerIdStr + " in NodeManager:" + nodeId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aec7cae..7b73aa6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -166,11 +167,22 @@ public void testHelpMessage() throws Exception { pw.println(" runing. Work with -logFiles to get other"); pw.println(" logs."); pw.println(" -help Displays help for all commands."); + pw.println(" -list_nodes Show the list of nodes that successfully"); + pw.println(" aggregated logs. This option can only be"); + pw.println(" used with finished applications."); pw.println(" -logFiles Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); pw.println(" all the log files for the container."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); + pw.println(" -show_meta_info Show the log metadata, including log-file"); + pw.println(" names, the size of the log files. You can"); + pw.println(" combine this with --containerId to get"); + pw.println(" log metadata for the specific container,"); + pw.println(" or with --nodeAddress to get log metadata"); + pw.println(" for all the containers on the specific"); + pw.println(" NodeManager. Currently, this option can"); + pw.println(" only be used for finished applications."); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -286,10 +298,10 @@ public void testFetchApplictionLogs() throws Exception { "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() }); assertTrue(exitCode == -1); - assertTrue(sysOutStream.toString().contains( + assertTrue(sysErrStream.toString().contains( "Logs for container " + containerId0.toString() + " are not present in this log-file.")); - sysOutStream.reset(); + sysErrStream.reset(); // uploaded two logs for container3. The first log is named as syslog. // The second one is named as stdout. @@ -322,6 +334,140 @@ public void testFetchApplictionLogs() throws Exception { } @Test (timeout = 15000) + public void testPrintContainerLogMetadata() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + FileSystem fs = FileSystem.get(configuration); + String rootLogDir = "target/LocalLogs"; + + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + List containerIds = new ArrayList(); + ContainerId containerId1 = ContainerId.newContainerId( + appAttemptId, 1); + ContainerId containerId2 = ContainerId.newContainerId( + appAttemptId, 2); + containerIds.add(containerId1); + containerIds.add(containerId2); + + List nodeIds = new ArrayList(); + NodeId nodeId = NodeId.newInstance("localhost", 1234); + nodeIds.add(nodeId); + nodeIds.add(nodeId); + + createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, + appId, containerIds, nodeIds); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-containerId", "container_0_0001_01_000001" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertFalse(sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-nodeAddress", "localhost" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-nodeAddress", "localhost", "-containerId", + "container_1234" }); + assertTrue(sysErrStream.toString().contains( + "The container container_1234 couldn't be found on the node " + + "specified: localhost")); + sysErrStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + fs.delete(new Path(rootLogDir), true); + } + + @Test (timeout = 15000) + public void testListNodeInfo() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + List containerIds = new ArrayList(); + ContainerId containerId1 = ContainerId.newContainerId( + appAttemptId, 1); + ContainerId containerId2 = ContainerId.newContainerId( + appAttemptId, 2); + containerIds.add(containerId1); + containerIds.add(containerId2); + + List nodeIds = new ArrayList(); + NodeId nodeId1 = NodeId.newInstance("localhost1", 1234); + NodeId nodeId2 = NodeId.newInstance("localhost2", 2345); + nodeIds.add(nodeId1); + nodeIds.add(nodeId2); + + String rootLogDir = "target/LocalLogs"; + FileSystem fs = FileSystem.get(configuration); + + createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, + appId, containerIds, nodeIds); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-list_nodes" }); + assertTrue(sysOutStream.toString().contains( + LogAggregationUtils.getNodeString(nodeId1))); + assertTrue(sysOutStream.toString().contains( + LogAggregationUtils.getNodeString(nodeId2))); + sysOutStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + fs.delete(new Path(rootLogDir), true); + } + + @Test (timeout = 15000) public void testFetchApplictionLogsHar() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); @@ -369,6 +515,46 @@ public void testFetchApplictionLogsHar() throws Exception { fs.delete(new Path(remoteLogRootDir), true); } + private void createContainerLogs(Configuration configuration, + String remoteLogRootDir, String rootLogDir, FileSystem fs, + ApplicationId appId, List containerIds, + List nodeIds) throws Exception { + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // create local logs + Path rootLogDirPath = new Path(rootLogDir); + if (fs.exists(rootLogDirPath)) { + fs.delete(rootLogDirPath, true); + } + assertTrue(fs.mkdirs(rootLogDirPath)); + + Path appLogsDir = new Path(rootLogDirPath, appId.toString()); + if (fs.exists(appLogsDir)) { + fs.delete(appLogsDir, true); + } + assertTrue(fs.mkdirs(appLogsDir)); + List rootLogDirs = Arrays.asList(rootLogDir); + List logTypes = new ArrayList(); + logTypes.add("syslog"); + // create container logs in localLogDir + for (ContainerId containerId : containerIds) { + createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes); + } + Path path = + new Path(remoteLogRootDir + ugi.getShortUserName() + + "/logs/application_0_0001"); + + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + for (int i=0; i logTypes) throws Exception { Path containerLogsDir = new Path(appLogsDir, containerId.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index c9453b3..9fae75a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -808,6 +808,25 @@ public static int readContainerLogsForALogType( } } + public static void readContainerMetaDataAndSkipData( + DataInputStream valueStream, PrintStream out) throws IOException { + + String fileType = valueStream.readUTF(); + String fileLengthStr = valueStream.readUTF(); + long fileLength = Long.parseLong(fileLengthStr); + out.print("LogType:"); + out.println(fileType); + out.print("LogLength:"); + out.println(fileLengthStr); + + long totalSkipped = 0; + long currSkipped = 0; + while (currSkipped != -1 && totalSkipped < fileLength) { + currSkipped = valueStream.skip(fileLength - totalSkipped); + totalSkipped += currSkipped; + } + } + public void close() { IOUtils.cleanup(LOG, scanner, reader, fsDataIStream); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..911d45d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -58,24 +58,10 @@ public int dumpAContainersLogs(String appId, String containerId, @VisibleForTesting public int dumpAContainersLogsForALogType(String appId, String containerId, String nodeId, String jobOwner, List logType) throws IOException { - Path remoteRootLogDir = new Path(getConf().get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); ApplicationId applicationId = ConverterUtils.toApplicationId(appId); - Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( - remoteRootLogDir, applicationId, jobOwner, - suffix); - RemoteIterator nodeFiles; - try { - Path qualifiedLogDir = - FileContext.getFileContext(getConf()).makeQualified( - remoteAppLogDir); - nodeFiles = - FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()) - .listStatus(remoteAppLogDir); - } catch (FileNotFoundException fnf) { - logDirNotExist(remoteAppLogDir.toString()); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + applicationId, jobOwner); + if (nodeFiles == null) { return -1; } boolean foundContainerLogs = false; @@ -194,22 +180,9 @@ public int dumpAContainerLogsForALogType(String containerIdStr, @Private public int dumpAllContainersLogs(ApplicationId appId, String appOwner, PrintStream out) throws IOException { - Path remoteRootLogDir = new Path(getConf().get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - String user = appOwner; - String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); - // TODO Change this to get a list of files from the LAS. - Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( - remoteRootLogDir, appId, user, logDirSuffix); - RemoteIterator nodeFiles; - try { - Path qualifiedLogDir = - FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); - nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), - getConf()).listStatus(remoteAppLogDir); - } catch (FileNotFoundException fnf) { - logDirNotExist(remoteAppLogDir.toString()); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { return -1; } boolean foundAnyLogs = false; @@ -257,12 +230,136 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, } } if (! foundAnyLogs) { - emptyLogDir(remoteAppLogDir.toString()); + emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString()); return -1; } return 0; } + @Private + public void printLogMetadata(ApplicationId appId, + String containerIdStr, String nodeId, String appOwner, + PrintStream out, PrintStream err) + throws IOException { + boolean getAllContainers = (containerIdStr == null); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundAnyLogs = false; + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + if (getAllContainers || (key.toString().equals(containerIdStr))) { + String containerString = + "\n\nContainer: " + key + " on " + + thisNodeFile.getPath().getName(); + out.println(containerString); + out.println("Log Upload Time:" + + thisNodeFile.getModificationTime()); + out.println(StringUtils.repeat("=", containerString.length())); + while (true) { + try { + LogReader.readContainerMetaDataAndSkipData(valueStream, out); + } catch (EOFException eof) { + break; + } + } + foundAnyLogs = true; + if (!getAllContainers) { + break; + } + } + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + if (!foundAnyLogs) { + if (containerIdStr != null && nodeId != null) { + err.println("The container " + containerIdStr + " couldn't be found " + + "on the node specified: " + nodeId); + } else if (nodeId != null) { + err.println("Can not find log metadata for any containers on " + + nodeId); + } else if (containerIdStr != null) { + err.println("Can not find log metadata for container: " + + containerIdStr); + } + } + } + + @Private + public void printNodesList(ApplicationId appId, String appOwner, + PrintStream out, PrintStream err) throws IOException { + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundNode = false; + StringBuilder sb = new StringBuilder(); + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + sb.append(thisNodeFile.getPath().getName() + "\n"); + foundNode = true; + } + if (!foundNode) { + err.println("No nodes found that aggregated logs for " + + "the application: " + appId); + } else { + out.println(sb.toString()); + } + } + + private RemoteIterator getRemoteNodeFileDir(ApplicationId appId, + String appOwner) throws IOException { + Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner); + RemoteIterator nodeFiles = null; + try { + Path qualifiedLogDir = + FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); + nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), + getConf()).listStatus(remoteAppLogDir); + } catch (FileNotFoundException fnf) { + logDirNotExist(remoteAppLogDir.toString()); + } + return nodeFiles; + } + + private Path getRemoteAppLogDir(ApplicationId appId, String appOwner) { + Path remoteRootLogDir = new Path(getConf().get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String user = appOwner; + String logDirSuffix = LogAggregationUtils + .getRemoteNodeLogDirSuffix(getConf()); + // TODO Change this to get a list of files from the LAS. + return LogAggregationUtils.getRemoteAppLogDir( + remoteRootLogDir, appId, user, logDirSuffix); + } + @Override public void setConf(Configuration conf) { this.conf = conf; @@ -274,16 +371,16 @@ public Configuration getConf() { } private static void containerLogNotFound(String containerId) { - System.out.println("Logs for container " + containerId + System.err.println("Logs for container " + containerId + " are not present in this log-file."); } private static void logDirNotExist(String remoteAppLogDir) { - System.out.println(remoteAppLogDir + " does not exist."); - System.out.println("Log aggregation has not completed or is not enabled."); + System.err.println(remoteAppLogDir + " does not exist."); + System.err.println("Log aggregation has not completed or is not enabled."); } private static void emptyLogDir(String remoteAppLogDir) { - System.out.println(remoteAppLogDir + " does not have any log files."); + System.err.println(remoteAppLogDir + " does not have any log files."); } }