diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index c049a74..59f0e54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -29,9 +29,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; @@ -107,6 +110,7 @@ private static final String CLIENT_RETRY_INTERVAL_OPTION = "client_retry_interval_ms"; public static final String HELP_CMD = "help"; + private static final String IGNORE_SIZE_LIMITS_OPTION = "ignore_size_limits"; private PrintStream outStream = System.out; private YarnClient yarnClient = null; @@ -115,6 +119,10 @@ private static final int DEFAULT_MAX_RETRIES = 30; private static final long DEFAULT_RETRY_INTERVAL = 1000; + private static final long LOG_SIZE_LIMIT_DEFAULT = 10 * 1024 * 1024 * 1024L; + + private long logSizeLeft = LOG_SIZE_LIMIT_DEFAULT; + @Private @VisibleForTesting ClientConnectionRetry connectionRetry; @@ -122,6 +130,7 @@ @Override public int run(String[] args) throws Exception { try { + logSizeLeft = getInitialSizeLimit(); yarnClient = createYarnClient(); webServiceClient = Client.create(); return runCommand(args); @@ -158,6 +167,7 @@ private int runCommand(String[] args) throws Exception { List amContainersList = new ArrayList(); String localDir = null; long bytes = Long.MAX_VALUE; + boolean ignoreSizeLimits = false; int maxRetries = DEFAULT_MAX_RETRIES; long retryInterval = DEFAULT_RETRY_INTERVAL; try { @@ -199,6 +209,9 @@ private int runCommand(String[] args) throws Exception { retryInterval = Long.parseLong(commandLine.getOptionValue( CLIENT_RETRY_INTERVAL_OPTION)); } + if (commandLine.hasOption(IGNORE_SIZE_LIMITS_OPTION)) { + ignoreSizeLimits = true; + } } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); @@ -306,6 +319,7 @@ private int runCommand(String[] args) throws Exception { logs.addAll(Arrays.asList(logFilesRegex)); } + ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null); @@ -324,15 +338,17 @@ private int runCommand(String[] args) throws Exception { // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, - logCliHelper, useRegex); + logCliHelper, useRegex, ignoreSizeLimits); } int resultCode = 0; if (containerIdStr != null) { - return fetchContainerLogs(request, logCliHelper, useRegex); + return fetchContainerLogs(request, logCliHelper, useRegex, + ignoreSizeLimits); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(request, logCliHelper, useRegex); + resultCode = fetchApplicationLogs(request, logCliHelper, useRegex, + ignoreSizeLimits); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -525,35 +541,16 @@ private ContainerLogFileInfo generatePerContainerLogFileInfoFromJSON( @VisibleForTesting public int printContainerLogsFromRunningApplication(Configuration conf, ContainerLogsRequest request, LogCLIHelpers logCliHelper, - boolean useRegex) throws IOException { + boolean useRegex, boolean ignoreSizeLimits) throws IOException { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); - String nodeHttpAddress = request.getNodeHttpAddress(); - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { - System.err.println("Can not get the logs for the container: " - + containerIdStr); - System.err.println("The node http address is required to get container " - + "logs for the Running application."); - return -1; - } String nodeId = request.getNodeId(); PrintStream out = LogToolUtils.createPrintStream(localDir, nodeId, containerIdStr); try { - Set matchedFiles = getMatchedContainerLogFiles(request, - useRegex); - if (matchedFiles.isEmpty()) { - System.err.println("Can not find any log file matching the pattern: " - + request.getLogTypes() + " for the container: " + containerIdStr - + " within the application: " + request.getAppId()); - return -1; - } - ContainerLogsRequest newOptions = new ContainerLogsRequest(request); - newOptions.setLogTypes(matchedFiles); - boolean foundAnyLogs = false; byte[] buffer = new byte[65536]; - for (String logFile : newOptions.getLogTypes()) { + for (String logFile : request.getLogTypes()) { InputStream is = null; try { ClientResponse response = getResponeFromNMWebService(conf, @@ -596,50 +593,6 @@ public int printContainerLogsFromRunningApplication(Configuration conf, } } - private int printContainerLogsForFinishedApplication( - ContainerLogsRequest request, LogCLIHelpers logCliHelper, - boolean useRegex) throws IOException { - ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper, useRegex); - if (newOptions == null) { - System.err.println("Can not find any log file matching the pattern: " - + request.getLogTypes() + " for the container: " - + request.getContainerId() + " within the application: " - + request.getAppId()); - return -1; - } - return logCliHelper.dumpAContainerLogsForLogType(newOptions); - } - - private int printContainerLogsForFinishedApplicationWithoutNodeId( - ContainerLogsRequest request, LogCLIHelpers logCliHelper, - boolean useRegex) throws IOException { - ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper, useRegex); - if (newOptions == null) { - System.err.println("Can not find any log file matching the pattern: " - + request.getLogTypes() + " for the container: " - + request.getContainerId() + " within the application: " - + request.getAppId()); - return -1; - } - return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( - newOptions); - } - - private int printAggregatedContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { - return printContainerLogsForFinishedApplication(request, - logCliHelper, useRegex); - } - - private int printAggregatedContainerLogsWithoutNodeId( - ContainerLogsRequest request, LogCLIHelpers logCliHelper, - boolean useRegex) throws IOException { - return printContainerLogsForFinishedApplicationWithoutNodeId(request, - logCliHelper, useRegex); - } - @Private @VisibleForTesting public ContainerReport getContainerReport(String containerIdStr) @@ -656,7 +609,8 @@ private boolean isApplicationFinished(YarnApplicationState appState) { private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, - LogCLIHelpers logCliHelper, boolean useRegex) throws Exception { + LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimits) + throws Exception { List amContainersList = null; List requests = new ArrayList(); @@ -718,10 +672,9 @@ private int printAMContainerLogs(Configuration conf, return -1; } + List candidates = new ArrayList<>(); if (amContainers.contains("ALL")) { - for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex); - } + candidates.addAll(requests); outStream.println(); outStream.println("Specified ALL for -am option. " + "Printed logs for all am containers."); @@ -729,12 +682,10 @@ private int printAMContainerLogs(Configuration conf, for (String amContainer : amContainers) { int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { - outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logCliHelper, useRegex); + candidates.add(requests.get(requests.size() - 1)); } else { if (amContainerId <= requests.size()) { - outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logCliHelper, useRegex); + candidates.add(requests.get(amContainerId - 1)); } else { System.err.println(String.format("ERROR: Specified AM containerId" + " (%s) exceeds the number of AM containers (%s).", @@ -744,12 +695,25 @@ private int printAMContainerLogs(Configuration conf, } } } + Map newOptions = new HashMap<>(); + if (request.isAppFinished()) { + newOptions = getMatchedLogTypesForFinishedApp(candidates, + logCliHelper, useRegex, ignoreSizeLimits); + } else { + newOptions = getMatchedLogTypesForRunningApp(candidates, useRegex, + ignoreSizeLimits); + } + for (Entry amRequest + : newOptions.entrySet()) { + outputAMContainerLogs(amRequest.getValue(), conf, logCliHelper, + useRegex, ignoreSizeLimits); + } return 0; } private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex) - throws Exception { + Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex, + boolean ignoreSizeLimits) throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -757,11 +721,10 @@ private void outputAMContainerLogs(ContainerLogsRequest request, if (request.isAppFinished()) { if (containerId != null && !containerId.isEmpty()) { if (nodeId != null && !nodeId.isEmpty()) { - printContainerLogsForFinishedApplication(request, - logCliHelper, useRegex); + logCliHelper.dumpAContainerLogsForLogType(request); } else { - printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper, useRegex); + logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( + request); } } } else { @@ -771,7 +734,7 @@ private void outputAMContainerLogs(ContainerLogsRequest request, .getContainerState(); request.setContainerState(containerState); printContainerLogsFromRunningApplication(conf, - request, logCliHelper, useRegex); + request, logCliHelper, useRegex, ignoreSizeLimits); } } } @@ -899,6 +862,11 @@ private Options createCommandOpts() { opts.addOption(CLIENT_RETRY_INTERVAL_OPTION, true, "Work with --client_max_retries to create a retry client. " + "The default value is 1000."); + opts.addOption(IGNORE_SIZE_LIMITS_OPTION, false, + "Use this option to ignore the total log size limits. By default, " + + "we only allow to fetch at most 10G logs. If the total log size is " + + "larger than 10G, the CLI would fail. The user could specify this " + + "option to ignore the size limit and fetch all logs."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); @@ -934,6 +902,7 @@ private Options createPrintOpts(Options commandOpts) { PER_CONTAINER_LOG_FILES_REGEX_OPTION)); printOpts.addOption(commandOpts.getOption(CLIENT_MAX_RETRY_OPTION)); printOpts.addOption(commandOpts.getOption(CLIENT_RETRY_INTERVAL_OPTION)); + printOpts.addOption(commandOpts.getOption(IGNORE_SIZE_LIMITS_OPTION)); return printOpts; } @@ -970,15 +939,15 @@ private Options createPrintOpts(Options commandOpts) { private int fetchAMContainerLogs(ContainerLogsRequest request, List amContainersList, LogCLIHelpers logCliHelper, - boolean useRegex) throws Exception { + boolean useRegex, boolean ignoreSizeLimits) throws Exception { return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper, useRegex); + logCliHelper, useRegex, ignoreSizeLimits); } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, - ClientHandlerException, UniformInterfaceException, JSONException { - int resultCode = 0; + LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimits) + throws IOException, ClientHandlerException, UniformInterfaceException, + JSONException { String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); String nodeAddress = request.getNodeId(); @@ -989,12 +958,20 @@ private int fetchContainerLogs(ContainerLogsRequest request, if (isAppFinished) { // if user specified "ALL" as the logFiles param, pass empty list // to logCliHelper so that it fetches all the logs + ContainerLogsRequest newOptions = getMatchedLogOptions( + request, logCliHelper, useRegex, ignoreSizeLimits); + if (newOptions == null) { + System.err.println("Can not find any log file matching the pattern: " + + request.getLogTypes() + " for the container: " + + request.getContainerId() + " within the application: " + + request.getAppId()); + return -1; + } if (nodeAddress != null && !nodeAddress.isEmpty()) { - return printContainerLogsForFinishedApplication( - request, logCliHelper, useRegex); + return logCliHelper.dumpAContainerLogsForLogType(newOptions); } else { - return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper, useRegex); + return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( + newOptions); } } String nodeHttpAddress = null; @@ -1020,13 +997,20 @@ private int fetchContainerLogs(ContainerLogsRequest request, } else { // for the case, we have already uploaded partial logs in HDFS int result = -1; - if (nodeAddress != null && !nodeAddress.isEmpty()) { - result = printAggregatedContainerLogs(request, - logCliHelper, useRegex); + ContainerLogsRequest newOptions = getMatchedLogOptions( + request, logCliHelper, useRegex, ignoreSizeLimits); + if (newOptions == null) { + System.err.println("Can not find any log file matching the pattern: " + + request.getLogTypes() + " for the container: " + + request.getContainerId() + " within the application: " + + request.getAppId()); } else { - result = printAggregatedContainerLogsWithoutNodeId(request, - logCliHelper, - useRegex); + if (nodeAddress != null && !nodeAddress.isEmpty()) { + result = logCliHelper.dumpAContainerLogsForLogType(newOptions); + } else { + result = logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( + newOptions); + } } if (result == -1) { System.err.println( @@ -1044,14 +1028,18 @@ private int fetchContainerLogs(ContainerLogsRequest request, // If the application is not in the final state, // we will provide the NodeHttpAddress and get the container logs // by calling NodeManager webservice. - resultCode = printContainerLogsFromRunningApplication(getConf(), request, - logCliHelper, useRegex); - return resultCode; + ContainerLogsRequest newRequest = getMatchedOptionForRunningApp( + request, useRegex, ignoreSizeLimits); + if (newRequest == null) { + return -1; + } + return printContainerLogsFromRunningApplication(getConf(), request, + logCliHelper, useRegex, ignoreSizeLimits); } private int fetchApplicationLogs(ContainerLogsRequest options, - LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, - YarnException { + LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimits) + throws IOException, YarnException { // If the application has finished, we would fetch the logs // from HDFS. // If the application is still running, we would get the full @@ -1060,7 +1048,7 @@ private int fetchApplicationLogs(ContainerLogsRequest options, int resultCode = -1; if (options.isAppFinished()) { ContainerLogsRequest newOptions = getMatchedLogOptions( - options, logCliHelper, useRegex); + options, logCliHelper, useRegex, ignoreSizeLimits); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + options.getLogTypes() + " for the application: " @@ -1072,9 +1060,17 @@ private int fetchApplicationLogs(ContainerLogsRequest options, } else { List containerLogRequests = getContainersLogRequestForRunningApplication(options); - for (ContainerLogsRequest container : containerLogRequests) { + + // get all matched container log types and check the total log size. + Map matchedLogTypes = + getMatchedLogTypesForRunningApp(containerLogRequests, + useRegex, ignoreSizeLimits); + + for (Entry container + : matchedLogTypes.entrySet()) { int result = printContainerLogsFromRunningApplication(getConf(), - container, logCliHelper, useRegex); + container.getValue(), logCliHelper, + useRegex, ignoreSizeLimits); if (result == 0) { resultCode = 0; } @@ -1104,37 +1100,54 @@ private String guessAppOwner(ApplicationReport appReport, private ContainerLogsRequest getMatchedLogOptions( ContainerLogsRequest request, LogCLIHelpers logCliHelper, - boolean useRegex) throws IOException { + boolean useRegex, boolean ignoreSizeLimits) throws IOException { ContainerLogsRequest newOptions = new ContainerLogsRequest(request); - if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) { - Set matchedFiles = new HashSet(); - if (!request.getLogTypes().contains("ALL")) { - Set files = logCliHelper.listContainerLogs(request); - matchedFiles = getMatchedLogFiles(request, files, useRegex); - if (matchedFiles.isEmpty()) { - return null; - } - } + Set matchedFiles = new HashSet(); + Set files = logCliHelper.listContainerLogs( + request); + matchedFiles = getMatchedLogFiles(request, files, useRegex, + ignoreSizeLimits); + if (matchedFiles.isEmpty()) { + return null; + } else { newOptions.setLogTypes(matchedFiles); + return newOptions; } - return newOptions; } private Set getMatchedLogFiles(ContainerLogsRequest options, - Collection candidate, boolean useRegex) throws IOException { + Collection candidate, boolean useRegex, + boolean ignoreSizeLimits) throws IOException { Set matchedFiles = new HashSet(); Set filePattern = options.getLogTypes(); - if (options.getLogTypes().contains("ALL")) { - return new HashSet(candidate); - } - for (String file : candidate) { - if (useRegex) { - if (isFileMatching(file, filePattern)) { - matchedFiles.add(file); + long size = options.getBytes(); + boolean getAll = options.getLogTypes().contains("ALL"); + Iterator iterator = candidate.iterator(); + while(iterator.hasNext()) { + boolean matchedFile = false; + ContainerLogFileInfo logInfo = iterator.next(); + if (getAll) { + matchedFile = true; + } else if (useRegex) { + if (isFileMatching(logInfo.getFileName(), filePattern)) { + matchedFile = true; } } else { - if (filePattern.contains(file)) { - matchedFiles.add(file); + if (filePattern.contains(logInfo.getFileName())) { + matchedFile = true; + } + } + if (matchedFile) { + matchedFiles.add(logInfo.getFileName()); + if (!ignoreSizeLimits) { + decrLogSizeLimits(Math.min( + Long.parseLong(logInfo.getFileSize()), size)); + if (getLogSizeLimitsLeft() < 0) { + throw new RuntimeException("The total log size is too large" + + "(more than 10G). Please specify a proper value " + + "--size option or if you really want to fetch all, please " + + "add --ignore_size_limits option."); + } } } } @@ -1297,18 +1310,19 @@ private void outputContainerLogMeta(String containerId, String nodeId, @VisibleForTesting public Set getMatchedContainerLogFiles(ContainerLogsRequest request, - boolean useRegex) throws IOException { + boolean useRegex, boolean ignoreSizeLimits) throws IOException { // fetch all the log files for the container // filter the log files based on the given -log_files pattern List> allLogFileInfos= getContainerLogFiles(getConf(), request.getContainerId(), request.getNodeHttpAddress()); - List fileNames = new ArrayList(); + List fileNames = new ArrayList< + ContainerLogFileInfo>(); for (Pair fileInfo : allLogFileInfos) { - fileNames.add(fileInfo.getKey().getFileName()); + fileNames.add(fileInfo.getKey()); } return getMatchedLogFiles(request, fileNames, - useRegex); + useRegex, ignoreSizeLimits); } @VisibleForTesting @@ -1452,4 +1466,92 @@ public boolean shouldRetryOn(Exception e) { // The method to indicate if we should retry given the incoming exception public abstract boolean shouldRetryOn(Exception e); } + + private long getLogSizeLimitsLeft() { + return this.logSizeLeft; + } + + private void decrLogSizeLimits(long used) { + this.logSizeLeft -= used; + } + + @Private + @VisibleForTesting + public long getInitialSizeLimit() { + return LOG_SIZE_LIMIT_DEFAULT; + } + + @Private + @VisibleForTesting + public ContainerLogsRequest getMatchedOptionForRunningApp( + ContainerLogsRequest container, boolean useRegex, + boolean ignoreSizeLimits) throws IOException { + String containerIdStr = container.getContainerId().toString(); + String nodeHttpAddress = container.getNodeHttpAddress(); + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + System.err.println("Can not get the logs for the container: " + + containerIdStr); + System.err.println("The node http address is required to get container " + + "logs for the Running application."); + return null; + } + + Set matchedFiles = getMatchedContainerLogFiles(container, + useRegex, ignoreSizeLimits); + if (matchedFiles.isEmpty()) { + System.err.println("Can not find any log file matching the pattern: " + + container.getLogTypes() + " for the container: " + containerIdStr + + " within the application: " + container.getAppId()); + return null; + } + container.setLogTypes(matchedFiles); + return container; + } + + @Private + @VisibleForTesting + public Map getMatchedLogTypesForRunningApp( + List containerLogRequests, boolean useRegex, + boolean ignoreSizeLimits) { + Map containerMatchedLog = new HashMap<>(); + for (ContainerLogsRequest container : containerLogRequests) { + try { + ContainerLogsRequest request = getMatchedOptionForRunningApp( + container, useRegex, ignoreSizeLimits); + if (request == null) { + continue; + } + containerMatchedLog.put(container.getContainerId(), request); + } catch(IOException ex) { + System.err.println(ex); + continue; + } + } + return containerMatchedLog; + } + + private Map getMatchedLogTypesForFinishedApp( + List containerLogRequests, + LogCLIHelpers logCliHelper, boolean useRegex, + boolean ignoreSizeLimits) { + Map containerMatchedLog = new HashMap<>(); + for (ContainerLogsRequest container : containerLogRequests) { + try { + ContainerLogsRequest request = getMatchedLogOptions(container, + logCliHelper, useRegex, ignoreSizeLimits); + if (request == null) { + System.err.println("Can not find any log file matching the pattern: " + + container.getLogTypes() + " for the container: " + + container.getContainerId() + " within the application: " + + container.getAppId()); + continue; + } + containerMatchedLog.put(container.getContainerId(), request); + } catch (IOException ex) { + System.err.println(ex); + continue; + } + } + return containerMatchedLog; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index e9d984e..098ee3a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Matchers.*; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; @@ -252,6 +253,16 @@ public void testHelpMessage() throws Exception { pw.println(" applicationId can be omitted"); pw.println(" -help Displays help for all"); pw.println(" commands."); + pw.println(" -ignore_size_limits Use this option to ignore"); + pw.println(" the total log size limits."); + pw.println(" By default, we only allow to"); + pw.println(" fetch at most 10G logs. If"); + pw.println(" the total log size is larger"); + pw.println(" than 10G, the CLI would"); + pw.println(" fail. The user could specify"); + pw.println(" this option to ignore the"); + pw.println(" size limit and fetch all"); + pw.println(" logs."); pw.println(" -list_nodes Show the list of nodes that"); pw.println(" successfully aggregated"); pw.println(" logs. This option can only"); @@ -562,8 +573,7 @@ public ContainerReport getContainerReport(String containerIdStr) containerId0.toString() }); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Logs for container " + containerId0.toString() - + " are not present in this log-file.")); + "Can not find any log file matching the pattern")); sysErrStream.reset(); // uploaded two logs for container3. The first log is named as syslog. @@ -749,7 +759,7 @@ public void testGetRunningContainerLogs() throws Exception { Set logsSet = new HashSet(); logsSet.add(fileName); doReturn(logsSet).when(cli).getMatchedContainerLogFiles( - any(ContainerLogsRequest.class), anyBoolean()); + any(ContainerLogsRequest.class), anyBoolean(), anyBoolean()); ClientResponse mockReponse = mock(ClientResponse.class); doReturn(ClientResponse.Status.OK).when(mockReponse) .getClientResponseStatus(); @@ -795,6 +805,7 @@ public void testFetchRunningApplicationLogs() throws Exception { doReturn(nodeId).when(mockContainerReport1).getAssignedNode(); doReturn("http://localhost:2345").when(mockContainerReport1) .getNodeHttpAddress(); + ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2); ContainerReport mockContainerReport2 = mock(ContainerReport.class); doReturn(containerId2).when(mockContainerReport2).getContainerId(); @@ -812,7 +823,19 @@ public void testFetchRunningApplicationLogs() throws Exception { LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient)); doReturn(0).when(cli).printContainerLogsFromRunningApplication( any(Configuration.class), any(ContainerLogsRequest.class), - any(LogCLIHelpers.class), anyBoolean()); + any(LogCLIHelpers.class), anyBoolean(), anyBoolean()); + Set logTypes = new HashSet<>(); + logTypes.add("ALL"); + ContainerLogsRequest mockContainer1 = mock(ContainerLogsRequest.class); + doReturn(logTypes).when(mockContainer1).getLogTypes(); + ContainerLogsRequest mockContainer2 = mock(ContainerLogsRequest.class); + doReturn(logTypes).when(mockContainer2).getLogTypes(); + Map matchedLogTypes = new HashMap<>(); + matchedLogTypes.put(containerId1.toString(), mockContainer1); + matchedLogTypes.put(containerId2.toString(), mockContainer2); + doReturn(matchedLogTypes).when(cli).getMatchedLogTypesForRunningApp( + anyListOf(ContainerLogsRequest.class), anyBoolean(), + anyBoolean()); cli.setConf(new YarnConfiguration()); int exitCode = cli.run(new String[] {"-applicationId", appId.toString()}); @@ -825,7 +848,7 @@ public void testFetchRunningApplicationLogs() throws Exception { // printContainerLogsFromRunningApplication twice verify(cli, times(2)).printContainerLogsFromRunningApplication( any(Configuration.class), logsRequestCaptor.capture(), - any(LogCLIHelpers.class), anyBoolean()); + any(LogCLIHelpers.class), anyBoolean(), anyBoolean()); // Verify that the log-type is "ALL" List capturedRequests = @@ -839,9 +862,12 @@ public void testFetchRunningApplicationLogs() throws Exception { mockYarnClient = createMockYarnClientWithException( YarnApplicationState.RUNNING, ugi.getShortUserName()); LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient)); + ContainerLogsRequest newOption = mock(ContainerLogsRequest.class); + doReturn(newOption).when(cli2).getMatchedOptionForRunningApp( + any(ContainerLogsRequest.class), anyBoolean(), anyBoolean()); doReturn(0).when(cli2).printContainerLogsFromRunningApplication( any(Configuration.class), any(ContainerLogsRequest.class), - any(LogCLIHelpers.class), anyBoolean()); + any(LogCLIHelpers.class), anyBoolean(), anyBoolean()); doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString( any(ContainerLogsRequest.class)); cli2.setConf(new YarnConfiguration()); @@ -851,7 +877,7 @@ public void testFetchRunningApplicationLogs() throws Exception { assertTrue(exitCode == 0); verify(cli2, times(1)).printContainerLogsFromRunningApplication( any(Configuration.class), logsRequestCaptor.capture(), - any(LogCLIHelpers.class), anyBoolean()); + any(LogCLIHelpers.class), anyBoolean(), anyBoolean()); } @Test (timeout = 15000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 97b78ec..887d92d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -350,10 +350,10 @@ public void closePrintStream(PrintStream out) { } @Private - public Set listContainerLogs(ContainerLogsRequest options) - throws IOException { + public Set listContainerLogs( + ContainerLogsRequest options) throws IOException { List containersLogMeta; - Set logTypes = new HashSet(); + Set logTypes = new HashSet(); try { containersLogMeta = getFileController(options.getAppId(), options.getAppOwner()).readAggregatedLogsMeta( @@ -364,7 +364,7 @@ public void closePrintStream(PrintStream out) { } for (ContainerLogMeta logMeta: containersLogMeta) { for (ContainerLogFileInfo fileInfo : logMeta.getContainerLogMeta()) { - logTypes.add(fileInfo.getFileName()); + logTypes.add(fileInfo); } } return logTypes; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index 989b326..92e3a08 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -268,6 +268,12 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, } while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); + if (thisNodeFile.getPath().getName().equals(appId + ".har")) { + Path p = new Path("har:///" + + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (nodeIdStr != null) { if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { continue;