diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index bbe636f..4c3519b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -85,8 +85,9 @@ private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; - private static final String SHOW_META_INFO = "show_meta_info"; private static final String LIST_NODES_OPTION = "list_nodes"; + private static final String LIST_CONTAINERS_OPTION = "list_containers"; + private static final String SHOW_CONTAINER_INFO = "show_container_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; public static final String HELP_CMD = "help"; @@ -109,8 +110,9 @@ public int run(String[] args) throws Exception { String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; - boolean showMetaInfo = false; boolean nodesList = false; + boolean containersList = false; + boolean showContainerInfo = false; String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; @@ -122,9 +124,10 @@ public int run(String[] args) throws Exception { nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); - showMetaInfo = commandLine.hasOption(SHOW_META_INFO); nodesList = commandLine.hasOption(LIST_NODES_OPTION); localDir = commandLine.getOptionValue(OUT_OPTION); + containersList = commandLine.hasOption(LIST_CONTAINERS_OPTION); + showContainerInfo = commandLine.hasOption(SHOW_CONTAINER_INFO); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -202,14 +205,17 @@ public int run(String[] args) throws Exception { isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes); - if (showMetaInfo) { - return showMetaInfo(request, logCliHelper); + if (showContainerInfo) { + return showContainerInfo(request, logCliHelper); } if (nodesList) { return showNodeLists(request, logCliHelper); } + if (containersList) { + return showContainerLists(request, logCliHelper); + } // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, @@ -572,12 +578,11 @@ private void outputAMContainerLogs(ContainerLogsRequest request, } } - private int showMetaInfo(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException { + private int showContainerInfo(ContainerLogsRequest request, + LogCLIHelpers logCliHelper) throws IOException, YarnException { if (!request.isAppFinished()) { - System.err.println("The -show_meta_info command can be only used " - + "with finished applications"); - return -1; + printContainerInfoFromRunningApplication(request); + return 0; } else { logCliHelper.printLogMetadata(request, System.out, System.err); return 0; @@ -596,6 +601,28 @@ private int showNodeLists(ContainerLogsRequest request, } } + private int showContainerLists(ContainerLogsRequest request, + LogCLIHelpers logCliHelper) throws IOException, YarnException { + if (!request.isAppFinished()) { + List reports = + getContainerReportsFromRunningApplication(request); + List filterReports = filterContainersInfo( + request, reports); + if (filterReports.isEmpty()) { + System.err.println("Can not find any containers for the applicaiton:" + + request.getAppId() + "."); + return -1; + } + for (ContainerReport report : filterReports) { + System.out.println(report.getContainerId()); + } + return 0; + } else { + logCliHelper.printContainersList(request, System.out, System.err); + return 0; + } + } + private Options createCommandOpts() { Options opts = new Options(); opts.addOption(HELP_CMD, false, "Displays help for all commands."); @@ -631,13 +658,15 @@ private Options createCommandOpts() { logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); - opts.addOption(SHOW_META_INFO, false, "Show the log metadata, " + opts.addOption(SHOW_CONTAINER_INFO, false, "Show the log metadata, " + "including log-file names, the size of the log files. " + "You can combine this with --containerId to get log metadata for " + "the specific container, or with --nodeAddress to get log metadata " - + "for all the containers on the specific NodeManager. " - + "Currently, this option can only be used for finished " - + "applications."); + + "for all the containers on the specific NodeManager."); + opts.addOption(LIST_CONTAINERS_OPTION, false, "Show the containerIds which" + + " belong to the specific Application. You can combine this with" + + " --nodeAddress to get containerIds for all the containers on this" + + " specific NodeManager."); opts.addOption(LIST_NODES_OPTION, false, "Show the list of nodes that successfully aggregated logs. " + "This option can only be used with finished applications."); @@ -665,8 +694,9 @@ private Options createPrintOpts(Options commandOpts) { printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION)); printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES)); - printOpts.addOption(commandOpts.getOption(SHOW_META_INFO)); printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION)); + printOpts.addOption(commandOpts.getOption(LIST_CONTAINERS_OPTION)); + printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); return printOpts; @@ -915,6 +945,29 @@ private boolean isFileMatching(String fileType, ContainerLogsRequest options) throws YarnException, IOException { List newOptionsList = new ArrayList(); + List reports = + getContainerReportsFromRunningApplication(options); + for (ContainerReport container : reports) { + ContainerLogsRequest newOptions = new ContainerLogsRequest(options); + newOptions.setContainerId(container.getContainerId().toString()); + newOptions.setNodeId(container.getAssignedNode().toString()); + newOptions.setNodeHttpAddress(container.getNodeHttpAddress() + .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); + // if we do not specify the value for CONTAINER_LOG_FILES option, + // we will only output syslog + List logFiles = newOptions.getLogTypes(); + if (logFiles == null || logFiles.isEmpty()) { + logFiles = Arrays.asList("syslog"); + newOptions.setLogTypes(logFiles); + } + newOptionsList.add(newOptions); + } + return newOptionsList; + } + + private List getContainerReportsFromRunningApplication( + ContainerLogsRequest options) throws YarnException, IOException { + List reports = new ArrayList(); YarnClient yarnClient = createYarnClient(); try { List attempts = @@ -922,25 +975,62 @@ private boolean isFileMatching(String fileType, for (ApplicationAttemptReport attempt : attempts) { List containers = yarnClient.getContainers( attempt.getApplicationAttemptId()); - for (ContainerReport container : containers) { - ContainerLogsRequest newOptions = new ContainerLogsRequest(options); - newOptions.setContainerId(container.getContainerId().toString()); - newOptions.setNodeId(container.getAssignedNode().toString()); - newOptions.setNodeHttpAddress(container.getNodeHttpAddress() - .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); - // if we do not specify the value for CONTAINER_LOG_FILES option, - // we will only output syslog - List logFiles = newOptions.getLogTypes(); - if (logFiles == null || logFiles.isEmpty()) { - logFiles = Arrays.asList("syslog"); - newOptions.setLogTypes(logFiles); - } - newOptionsList.add(newOptions); - } + reports.addAll(containers); } - return newOptionsList; + return reports; } finally { yarnClient.close(); } } + + // filter the containerReports based on the nodeId and ContainerId + private List filterContainersInfo( + ContainerLogsRequest options, List containers) { + List filterReports = new ArrayList( + containers); + String nodeId = options.getNodeId(); + String containerId = options.getContainerId(); + + // filter the reports based on the containerId + if (containerId != null && !containerId.isEmpty()) { + for(ContainerReport report : containers) { + if (!report.getContainerId().toString() + .equalsIgnoreCase(containerId)) { + filterReports.remove(report); + } + } + } + + // filter the reports based on the nodeId + if (nodeId != null && !nodeId.isEmpty()) { + for(ContainerReport report : containers) { + if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) { + filterReports.remove(report); + } + } + } + return filterReports; + } + + private void printContainerInfoFromRunningApplication( + ContainerLogsRequest options) throws YarnException, IOException { + List reports = + getContainerReportsFromRunningApplication(options); + List filteredReports = filterContainersInfo( + options, reports); + for (ContainerReport report : filteredReports) { + String nodeId = report.getAssignedNode().toString(); + String nodeHttpAddress = report.getNodeHttpAddress(); + String containerId = report.getContainerId().toString(); + String containerString = + "\n\nContainer: " + containerId + " on " + nodeId; + System.out.println(containerString); + System.out.println(StringUtils.repeat("=", containerString.length())); + List allLogs= getContainerLogFiles(getConf(), + containerId, nodeHttpAddress); + for (String logs : allLogs) { + System.out.println(logs); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index db40b50..e218f04 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -182,6 +182,11 @@ public void testHelpMessage() throws Exception { pw.println(" runing. Work with -logFiles to get other"); pw.println(" logs."); pw.println(" -help Displays help for all commands."); + pw.println(" -list_containers Show the containerIds which belong to the"); + pw.println(" specific Application. You can combine"); + pw.println(" this with --nodeAddress to get"); + pw.println(" containerIds for all the containers on"); + pw.println(" this specific NodeManager."); pw.println(" -list_nodes Show the list of nodes that successfully"); pw.println(" aggregated logs. This option can only be"); pw.println(" used with finished applications."); @@ -195,14 +200,13 @@ public void testHelpMessage() throws Exception { pw.println(" container logs. The container logs will"); pw.println(" be stored based on the node the container"); pw.println(" ran on."); - pw.println(" -show_meta_info Show the log metadata, including log-file"); + pw.println(" -show_container_info Show the log metadata, including log-file"); pw.println(" names, the size of the log files. You can"); pw.println(" combine this with --containerId to get"); pw.println(" log metadata for the specific container,"); pw.println(" or with --nodeAddress to get log metadata"); pw.println(" for all the containers on the specific"); - pw.println(" NodeManager. Currently, this option can"); - pw.println(" only be used for finished applications."); + pw.println(" NodeManager."); pw.println(" -size Prints the log file's first 'n' bytes or"); pw.println(" the last 'n' bytes. Use negative values"); pw.println(" as bytes to read from the end and"); @@ -791,7 +795,7 @@ public void testPrintContainerLogMetadata() throws Exception { cli.setConf(configuration); cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info" }); + "-show_container_info" }); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( @@ -803,7 +807,7 @@ public void testPrintContainerLogMetadata() throws Exception { sysOutStream.reset(); cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-containerId", "container_0_0001_01_000001" }); + "-show_container_info", "-containerId", "container_0_0001_01_000001" }); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertFalse(sysOutStream.toString().contains( @@ -815,7 +819,7 @@ public void testPrintContainerLogMetadata() throws Exception { sysOutStream.reset(); cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-nodeAddress", "localhost" }); + "-show_container_info", "-nodeAddress", "localhost" }); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( @@ -827,13 +831,29 @@ public void testPrintContainerLogMetadata() throws Exception { sysOutStream.reset(); cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-nodeAddress", "localhost", "-containerId", + "-show_container_info", "-nodeAddress", "localhost", "-containerId", "container_1234" }); assertTrue(sysErrStream.toString().contains( "The container container_1234 couldn't be found on the node " + "specified: localhost")); sysErrStream.reset(); + cli.run(new String[] { "-applicationId", appId.toString(), + "-list_containers" }); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000001")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000002")); + sysOutStream.reset(); + + cli.run(new String[] { "-applicationId", appId.toString(), + "-list_containers", "-nodeAddress", "localhost" }); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000001")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000002")); + sysOutStream.reset(); + fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 3811054..537b8b4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -502,6 +502,59 @@ public void printNodesList(ContainerLogsRequest options, } } + @Private + public void printContainersList(ContainerLogsRequest options, + PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); + String nodeId = options.getNodeId(); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundAnyLogs = false; + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + out.println(key); + foundAnyLogs = true; + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + if (!foundAnyLogs) { + if (nodeId != null) { + err.println("Can not find information for any containers on " + + nodeId); + } else { + err.println("Can not find any container information for " + + "the application: " + appId); + } + } + } + private RemoteIterator getRemoteNodeFileDir(ApplicationId appId, String appOwner) throws IOException { Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner);