diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 12338fd..cc6b738 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -90,6 +90,7 @@ = "show_container_log_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; + private static final String REGEX_OPTION = "regex"; public static final String HELP_CMD = "help"; private PrintStream outStream = System.out; private YarnClient yarnClient = null; @@ -126,6 +127,7 @@ private int runCommand(String[] args) throws Exception { boolean nodesList = false; boolean showApplicationLogInfo = false; boolean showContainerLogInfo = false; + boolean useRegex = false; String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; @@ -142,6 +144,7 @@ private int runCommand(String[] args) throws Exception { showApplicationLogInfo = commandLine.hasOption( SHOW_APPLICATION_LOG_INFO); showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); + useRegex = commandLine.hasOption(REGEX_OPTION); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -268,15 +271,15 @@ private int runCommand(String[] args) throws Exception { // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } int resultCode = 0; if (containerIdStr != null) { - return fetchContainerLogs(request, logCliHelper); + return fetchContainerLogs(request, logCliHelper, useRegex); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(request, logCliHelper); + resultCode = fetchApplicationLogs(request, logCliHelper, useRegex); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -418,8 +421,8 @@ private boolean fetchAllLogFiles(String[] logFiles) { @Private @VisibleForTesting public int printContainerLogsFromRunningApplication(Configuration conf, - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); String nodeHttpAddress = request.getNodeHttpAddress(); @@ -442,7 +445,8 @@ public int printContainerLogsFromRunningApplication(Configuration conf, for (PerLogFileInfo fileInfo : allLogFileInfos) { fileNames.add(fileInfo.getFileName()); } - List matchedFiles = getMatchedLogFiles(request, fileNames); + List matchedFiles = getMatchedLogFiles(request, fileNames, + useRegex); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + containerIdStr @@ -504,10 +508,10 @@ public int printContainerLogsFromRunningApplication(Configuration conf, } private int printContainerLogsForFinishedApplication( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -519,10 +523,10 @@ private int printContainerLogsForFinishedApplication( } private int printContainerLogsForFinishedApplicationWithoutNodeId( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -550,7 +554,7 @@ private boolean isApplicationFinished(YarnApplicationState appState) { private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, - LogCLIHelpers logCliHelper) throws Exception { + LogCLIHelpers logCliHelper, boolean useRegex) throws Exception { List amContainersList = null; List requests = new ArrayList(); @@ -614,7 +618,7 @@ private int printAMContainerLogs(Configuration conf, if (amContainers.contains("ALL")) { for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logCliHelper); + outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex); } outStream.println(); outStream.println("Specified ALL for -am option. " @@ -624,11 +628,11 @@ private int printAMContainerLogs(Configuration conf, int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { if (amContainerId <= requests.size()) { outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { System.err.println(String.format("ERROR: Specified AM containerId" + " (%s) exceeds the number of AM containers (%s).", @@ -642,7 +646,8 @@ private int printAMContainerLogs(Configuration conf, } private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, LogCLIHelpers logCliHelper) throws Exception { + Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex) + throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -651,10 +656,10 @@ private void outputAMContainerLogs(ContainerLogsRequest request, if (containerId != null && !containerId.isEmpty()) { if (nodeId != null && !nodeId.isEmpty()) { printContainerLogsForFinishedApplication(request, - logCliHelper); + logCliHelper, useRegex); } else { printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } } else { @@ -664,7 +669,7 @@ private void outputAMContainerLogs(ContainerLogsRequest request, .getContainerState(); request.setContainerState(containerState); printContainerLogsFromRunningApplication(conf, - request, logCliHelper); + request, logCliHelper, useRegex); } } } @@ -747,12 +752,15 @@ private Options createCommandOpts() { opts.addOption(amOption); Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true, "Specify comma-separated value " - + "to get specified container log files. Use \"ALL\" to fetch all the " - + "log files for the container. It also supports Java Regex."); + + "to get exact matched log files. Use \"ALL\" or \".*\"to " + + "fetch all the log files for the container. Specific -regex " + + "for using java regex to find matched log files."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(REGEX_OPTION, false, "Work with -log_files to find " + + "matched files by using java regex."); opts.addOption(SHOW_CONTAINER_LOG_INFO, false, "Show the container log metadata, " + "including log-file names, the size of the log files. " @@ -795,6 +803,7 @@ private Options createPrintOpts(Options commandOpts) { printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); + printOpts.addOption(commandOpts.getOption(REGEX_OPTION)); return printOpts; } @@ -830,14 +839,14 @@ private Options createPrintOpts(Options commandOpts) { } private int fetchAMContainerLogs(ContainerLogsRequest request, - List amContainersList, LogCLIHelpers logCliHelper) - throws Exception { + List amContainersList, LogCLIHelpers logCliHelper, + boolean useRegex) throws Exception { return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -851,10 +860,10 @@ private int fetchContainerLogs(ContainerLogsRequest request, // to logCliHelper so that it fetches all the logs if (nodeAddress != null && !nodeAddress.isEmpty()) { return printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } else { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } String nodeHttpAddress = null; @@ -876,7 +885,7 @@ private int fetchContainerLogs(ContainerLogsRequest request, } catch (IOException | YarnException ex) { if (isAppFinished) { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } else { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appIdStr @@ -893,18 +902,19 @@ private int fetchContainerLogs(ContainerLogsRequest request, // by calling NodeManager webservice. if (!isAppFinished) { resultCode = printContainerLogsFromRunningApplication(getConf(), request, - logCliHelper); + logCliHelper, useRegex); } else { // If the application is in the final state, we will directly // get the container logs from HDFS. resultCode = printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } return resultCode; } private int fetchApplicationLogs(ContainerLogsRequest options, - LogCLIHelpers logCliHelper) throws IOException, YarnException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, + YarnException { // If the application has finished, we would fetch the logs // from HDFS. // If the application is still running, we would get the full @@ -913,7 +923,7 @@ private int fetchApplicationLogs(ContainerLogsRequest options, int resultCode = -1; if (options.isAppFinished()) { ContainerLogsRequest newOptions = getMatchedLogOptions( - options, logCliHelper); + options, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + options.getLogTypes() + " for the application: " @@ -927,7 +937,7 @@ private int fetchApplicationLogs(ContainerLogsRequest options, getContainersLogRequestForRunningApplication(options); for (ContainerLogsRequest container : containerLogRequests) { int result = printContainerLogsFromRunningApplication(getConf(), - container, logCliHelper); + container, logCliHelper, useRegex); if (result == 0) { resultCode = 0; } @@ -956,14 +966,14 @@ private String guessAppOwner(ApplicationReport appReport, } private ContainerLogsRequest getMatchedLogOptions( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = new ContainerLogsRequest(request); if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) { List matchedFiles = new ArrayList(); if (!request.getLogTypes().contains(".*")) { Set files = logCliHelper.listContainerLogs(request); - matchedFiles = getMatchedLogFiles(request, files); + matchedFiles = getMatchedLogFiles(request, files, useRegex); if (matchedFiles.isEmpty()) { return null; } @@ -974,13 +984,24 @@ private ContainerLogsRequest getMatchedLogOptions( } private List getMatchedLogFiles(ContainerLogsRequest options, - Collection candidate) throws IOException { + Collection candidate, boolean useRegex) throws IOException { List matchedFiles = new ArrayList(); List filePattern = options.getLogTypes(); + boolean fetchAll = fetchAllLogFiles( + filePattern.toArray(new String[filePattern.size()])); for (String file : candidate) { - if (isFileMatching(file, filePattern)) { + if (fetchAll) { matchedFiles.add(file); } + if (useRegex) { + if (isFileMatching(file, filePattern)) { + matchedFiles.add(file); + } + } else { + if (filePattern.contains(file)) { + matchedFiles.add(file); + } + } } return matchedFiles; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 329d856..4637022 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -211,14 +212,17 @@ public void testHelpMessage() throws Exception { pw.println(" aggregated logs. This option can only be"); pw.println(" used with finished applications."); pw.println(" -log_files Specify comma-separated value to get"); - pw.println(" specified container log files. Use \"ALL\""); - pw.println(" to fetch all the log files for the"); - pw.println(" container. It also supports Java Regex."); + pw.println(" exact matched log files. Use \"ALL\" or"); + pw.println(" \".*\"to fetch all the log files for the"); + pw.println(" container. Specific -regex for using java"); + pw.println(" regex to find matched log files."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.println(" -out Local directory for storing individual"); pw.println(" container logs. The container logs will"); pw.println(" be stored based on the node the container"); pw.println(" ran on."); + pw.println(" -regex Work with -log_files to find matched"); + pw.println(" files by using java regex."); pw.println(" -show_application_log_info Show the containerIds which belong to the"); pw.println(" specific Application. You can combine"); pw.println(" this with --nodeAddress to get"); @@ -286,6 +290,7 @@ public void testFetchFinishedApplictionLogs() throws Exception { // create two logs for container3 in localLogDir logTypes.add("stdout"); + logTypes.add("stdout1234"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes); Path path = @@ -336,6 +341,8 @@ public ContainerReport getContainerReport(String containerIdStr) logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -349,11 +356,28 @@ public ContainerReport getContainerReport(String containerIdStr) logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", "std*"}); + "-log_files", "stdout"}); + assertTrue(exitCode == 0); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", "std*", "-regex"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -363,6 +387,8 @@ public ContainerReport getContainerReport(String containerIdStr) logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -568,7 +594,7 @@ public void testFetchRunningApplicationLogs() throws Exception { LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient)); doReturn(0).when(cli).printContainerLogsFromRunningApplication( any(Configuration.class), any(ContainerLogsRequest.class), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); cli.setConf(new YarnConfiguration()); int exitCode = cli.run(new String[] {"-applicationId", appId.toString()}); @@ -581,7 +607,7 @@ public void testFetchRunningApplicationLogs() throws Exception { // printContainerLogsFromRunningApplication twice verify(cli, times(2)).printContainerLogsFromRunningApplication( any(Configuration.class), logsRequestCaptor.capture(), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); // Verify that the log-type is * List capturedRequests =