diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..b26841e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -78,6 +78,8 @@ private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; + private static final String INFO_OPTION = "show_meta_info"; + private static final String LIST_NODES_OPTION = "list_nodes"; public static final String HELP_CMD = "help"; @Override @@ -117,6 +119,15 @@ public int run(String[] args) throws Exception { logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(INFO_OPTION, false, "List Container log metadata," + + "including log file names, the size of the log files. Specify " + + "--containerId to get log metadata for the specific container. " + + "Or specify --nodeAddress to get all container log metadata for " + + "the specific nodemanager. Currently, this option can only be used " + + "when this applicaiton is finished."); + opts.addOption(LIST_NODES_OPTION, false, + "Show which nodes successfully aggregate logs. This option can only " + + "be used when this application is finished."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); @@ -131,6 +142,8 @@ public int run(String[] args) throws Exception { printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); + printOpts.addOption(opts.getOption(INFO_OPTION)); + printOpts.addOption(opts.getOption(LIST_NODES_OPTION)); if (args.length < 1) { printHelpMessage(printOpts); @@ -146,6 +159,8 @@ public int run(String[] args) throws Exception { String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; + boolean ls = false; + boolean nodeList = false; String[] logFiles = null; List amContainersList = new ArrayList(); try { @@ -155,6 +170,8 @@ public int run(String[] args) throws Exception { nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); + ls = commandLine.hasOption(INFO_OPTION); + nodeList = commandLine.hasOption(LIST_NODES_OPTION); if (getAMContainerLogs) { String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); for (String am : amContainers) { @@ -212,6 +229,7 @@ public int run(String[] args) throws Exception { appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); } + boolean getAppState = true; YarnApplicationState appState = YarnApplicationState.NEW; try { appState = getApplicationState(appId); @@ -224,6 +242,30 @@ public int run(String[] args) throws Exception { } catch (IOException | YarnException e) { System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem."); + getAppState = false; + } + + if (ls) { + if (!isApplicationFinished(appState) && getAppState) { + System.out.println("The -show_meta_info command can be only used " + + "when the Application is finished"); + return -1; + } else { + logCliHelper.printContainerLogMetadata(appId, containerIdStr, + nodeAddress, appOwner, System.out); + return 0; + } + } + + if (nodeList) { + if (!isApplicationFinished(appState) && getAppState) { + System.out.println("The -list-nodes command can be only used " + + "when the Application is finished"); + return -1; + } else { + logCliHelper.printNodeList(appId, appOwner, System.out); + return 0; + } } // To get am logs diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aec7cae..17a2def 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -166,11 +166,22 @@ public void testHelpMessage() throws Exception { pw.println(" runing. Work with -logFiles to get other"); pw.println(" logs."); pw.println(" -help Displays help for all commands."); + pw.println(" -list_nodes Show which nodes successfully aggregate"); + pw.println(" logs. This option can only be used when"); + pw.println(" this application is finished."); pw.println(" -logFiles Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); pw.println(" all the log files for the container."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); + pw.println(" -show_meta_info List Container log metadata,including log"); + pw.println(" file names, the size of the log files."); + pw.println(" Specify --containerId to get log metadata"); + pw.println(" for the specific container. Or specify"); + pw.println(" --nodeAddress to get all container log"); + pw.println(" metadata for the specific nodemanager."); + pw.println(" Currently, this option can only be used"); + pw.println(" when this applicaiton is finished."); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -322,6 +333,140 @@ public void testFetchApplictionLogs() throws Exception { } @Test (timeout = 15000) + public void testPrintContainerLogMetadata() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + FileSystem fs = FileSystem.get(configuration); + String rootLogDir = "target/LocalLogs"; + + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + List containerIds = new ArrayList(); + ContainerId containerId1 = ContainerId.newContainerId( + appAttemptId, 1); + ContainerId containerId2 = ContainerId.newContainerId( + appAttemptId, 2); + containerIds.add(containerId1); + containerIds.add(containerId2); + + List nodeIds = new ArrayList(); + NodeId nodeId = NodeId.newInstance("localhost", 1234); + nodeIds.add(nodeId); + nodeIds.add(nodeId); + + createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, + appId, containerIds, nodeIds); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-containerId", "container_0_0001_01_000001" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertTrue(!sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-nodeAddress", "localhost" }); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000001 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "Container: container_0_0001_01_000002 on localhost_")); + assertTrue(sysOutStream.toString().contains( + "LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "LogLength:43")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-show_meta_info", "-nodeAddress", "localhost", "-containerId", + "container_1234" }); + assertTrue(sysOutStream.toString().contains( + "The container container_1234 couldn't be found on the node " + + "specified: localhost")); + sysOutStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + fs.delete(new Path(rootLogDir), true); + } + + @Test (timeout = 15000) + public void testListNodeInfo() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + List containerIds = new ArrayList(); + ContainerId containerId1 = ContainerId.newContainerId( + appAttemptId, 1); + ContainerId containerId2 = ContainerId.newContainerId( + appAttemptId, 2); + containerIds.add(containerId1); + containerIds.add(containerId2); + + List nodeIds = new ArrayList(); + NodeId nodeId1 = NodeId.newInstance("localhost1", 1234); + NodeId nodeId2 = NodeId.newInstance("localhost2", 2345); + nodeIds.add(nodeId1); + nodeIds.add(nodeId2); + + String rootLogDir = "target/LocalLogs"; + FileSystem fs = FileSystem.get(configuration); + + createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, + appId, containerIds, nodeIds); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-list_nodes" }); + assertTrue(sysOutStream.toString().contains( + LogAggregationUtils.getNodeString(nodeId1))); + assertTrue(sysOutStream.toString().contains( + LogAggregationUtils.getNodeString(nodeId2))); + sysOutStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + fs.delete(new Path(rootLogDir), true); + } + + @Test (timeout = 15000) public void testFetchApplictionLogsHar() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); @@ -369,6 +514,46 @@ public void testFetchApplictionLogsHar() throws Exception { fs.delete(new Path(remoteLogRootDir), true); } + private void createContainerLogs(Configuration configuration, + String remoteLogRootDir, String rootLogDir, FileSystem fs, + ApplicationId appId, List containerIds, + List nodeIds) throws Exception { + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // create local logs + Path rootLogDirPath = new Path(rootLogDir); + if (fs.exists(rootLogDirPath)) { + fs.delete(rootLogDirPath, true); + } + assertTrue(fs.mkdirs(rootLogDirPath)); + + Path appLogsDir = new Path(rootLogDirPath, appId.toString()); + if (fs.exists(appLogsDir)) { + fs.delete(appLogsDir, true); + } + assertTrue(fs.mkdirs(appLogsDir)); + List rootLogDirs = Arrays.asList(rootLogDir); + List logTypes = new ArrayList(); + logTypes.add("syslog"); + // create container logs in localLogDir + for (ContainerId containerId : containerIds) { + createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes); + } + Path path = + new Path(remoteLogRootDir + ugi.getShortUserName() + + "/logs/application_0_0001"); + + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + for (int i=0; i logTypes) throws Exception { Path containerLogsDir = new Path(appLogsDir, containerId.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index c9453b3..84b6b9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -808,6 +808,26 @@ public static int readContainerLogsForALogType( } } + public static int listContainerLogs(DataInputStream valueStream, + PrintStream out) throws IOException { + + String fileType = valueStream.readUTF(); + String fileLengthStr = valueStream.readUTF(); + long fileLength = Long.parseLong(fileLengthStr); + out.print("LogType:"); + out.println(fileType); + out.print("LogLength:"); + out.println(fileLengthStr); + + long totalSkipped = 0; + long currSkipped = 0; + while (currSkipped != -1 && totalSkipped < fileLength) { + currSkipped = valueStream.skip(fileLength - totalSkipped); + totalSkipped += currSkipped; + } + return 0; + } + public void close() { IOUtils.cleanup(LOG, scanner, reader, fsDataIStream); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..0efd55e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; @@ -263,6 +264,125 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, return 0; } + @Private + public void printContainerLogMetadata(ApplicationId appId, + String containerIdStr, String nodeId, String appOwner, PrintStream out) + throws IOException { + boolean getAllContainers = (containerIdStr == null); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundAnyLogs = false; + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + if (getAllContainers || (key.toString().equals(containerIdStr))) { + String containerString = + "\n\nContainer: " + key + " on " + + thisNodeFile.getPath().getName(); + out.println(containerString); + out.println("Log Upload Time:" + + thisNodeFile.getModificationTime()); + out.println(StringUtils.repeat("=", containerString.length())); + while (true) { + try { + LogReader.listContainerLogs(valueStream, out); + } catch (EOFException eof) { + break; + } + } + foundAnyLogs = true; + } + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + if (!foundAnyLogs) { + if (containerIdStr != null && nodeId != null) { + out.println("The container " + containerIdStr + " couldn't be found " + + "on the node specified: " + nodeId); + } else if (nodeId != null) { + out.println("Can not find log metadata for any containers on " + + nodeId); + } else if (containerIdStr != null) { + out.println("Can not find log metadata for container: " + + containerIdStr); + } + } + } + + @Private + public void printNodeList(ApplicationId appId, String appOwner, + PrintStream out) throws IOException { + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundNode = false; + StringBuilder sb = new StringBuilder(); + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + sb.append(thisNodeFile.getPath().getName() + "\n"); + foundNode = true; + } + if (!foundNode) { + out.println("No nodes could successfully aggregate logs for " + + "the applicaiton: " + appId); + } else { + String appString = "\n\nApplicaiton: " + appId; + out.println(appString); + out.println(StringUtils.repeat("=", appString.length())); + out.println(sb.toString()); + } + } + + private RemoteIterator getRemoteNodeFileDir(ApplicationId appId, + String appOwner) throws IOException { + Path remoteRootLogDir = new Path(getConf().get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String user = appOwner; + String logDirSuffix = LogAggregationUtils + .getRemoteNodeLogDirSuffix(getConf()); + // TODO Change this to get a list of files from the LAS. + Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( + remoteRootLogDir, appId, user, logDirSuffix); + RemoteIterator nodeFiles = null; + try { + Path qualifiedLogDir = + FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); + nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), + getConf()).listStatus(remoteAppLogDir); + } catch (FileNotFoundException fnf) { + logDirNotExist(remoteAppLogDir.toString()); + } + return nodeFiles; + } + @Override public void setConf(Configuration conf) { this.conf = conf;