diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index d3b44a7..4e95c9c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -19,6 +19,11 @@ package org.apache.hadoop.yarn.client.cli; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.ws.rs.core.MediaType; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -27,6 +32,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -35,13 +41,25 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; @Public @Evolving @@ -51,6 +69,8 @@ private static final String APPLICATION_ID_OPTION = "applicationId"; private static final String NODE_ADDRESS_OPTION = "nodeAddress"; private static final String APP_OWNER_OPTION = "appOwner"; + private static final String AM_CONTAINER_OPTION = "am"; + private static final String CONTAINER_LOG_FILES = "logFiles"; public static final String HELP_CMD = "help"; @Override @@ -62,22 +82,45 @@ public int run(String[] args) throws Exception { new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); appIdOpt.setRequired(true); opts.addOption(appIdOpt); - opts.addOption(CONTAINER_ID_OPTION, true, - "ContainerId (must be specified if node address is specified)"); + opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. " + + "By default, it will only print syslog if the application is runing." + + " Work with -logFiles to get other logs."); opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " - + "nodename:port (must be specified if container id is specified)"); + + "nodename:port"); opts.addOption(APP_OWNER_OPTION, true, "AppOwner (assumed to be current user if not specified)"); + Option amOption = new Option(AM_CONTAINER_OPTION, true, + "Prints the AM Container logs for this application. " + + "Specify comma-separated value to get logs for related AM Container. " + + "To get logs for all AM Containers, use -am ALL. " + + "To get logs for the latest AM Container, use -am -1. " + + "By default, it will only print out syslog. Work with -logFiles " + + "to get other logs"); + amOption.setValueSeparator(','); + amOption.setArgs(Option.UNLIMITED_VALUES); + amOption.setArgName("AM Containers"); + opts.addOption(amOption); + Option logFileOpt = new Option(CONTAINER_LOG_FILES, true, + "Work with -am/-containerId and specify comma-separated value " + + "to get specified Container log files"); + logFileOpt.setValueSeparator(','); + logFileOpt.setArgs(Option.UNLIMITED_VALUES); + logFileOpt.setArgName("Log File Name"); + opts.addOption(logFileOpt); + opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); + opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers"); Options printOpts = new Options(); printOpts.addOption(opts.getOption(HELP_CMD)); printOpts.addOption(opts.getOption(CONTAINER_ID_OPTION)); printOpts.addOption(opts.getOption(NODE_ADDRESS_OPTION)); printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); + printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); + printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); if (args.length < 1) { printHelpMessage(printOpts); @@ -92,12 +135,46 @@ public int run(String[] args) throws Exception { String containerIdStr = null; String nodeAddress = null; String appOwner = null; + boolean getAMContainerLogs = false; + String[] logFiles = null; + List amContainersList = new ArrayList(); try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); + getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); + if (getAMContainerLogs) { + String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); + for (String am : amContainers) { + boolean errorInput = false; + if (!am.trim().equalsIgnoreCase("ALL")) { + try { + int id = Integer.parseInt(am.trim()); + if (id != -1 && id <= 0) { + errorInput = true; + } + } catch (NumberFormatException ex) { + errorInput = true; + } + if (errorInput) { + System.err.println( + "Invalid input for option -am. Valid inputs are 'ALL', -1 " + + "and any other integer which is larger than 0."); + printHelpMessage(printOpts); + return -1; + } + amContainersList.add(am.trim()); + } else { + amContainersList.add("ALL"); + break; + } + } + } + if (commandLine.hasOption(CONTAINER_LOG_FILES)) { + logFiles = commandLine.getOptionValues(CONTAINER_LOG_FILES); + } } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); @@ -118,64 +195,112 @@ public int run(String[] args) throws Exception { return -1; } + LogCLIHelpers logCliHelper = new LogCLIHelpers(); + logCliHelper.setConf(getConf()); + + if (appOwner == null || appOwner.isEmpty()) { + appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); + } + + YarnApplicationState appState = YarnApplicationState.NEW; try { - int resultCode = verifyApplicationState(appId); - if (resultCode != 0) { + appState = getApplicationState(appId); + if (appState == YarnApplicationState.NEW + || appState == YarnApplicationState.NEW_SAVING + || appState == YarnApplicationState.SUBMITTED) { System.out.println("Logs are not avaiable right now."); - return resultCode; + return -1; } - } catch (Exception e) { + } catch (IOException | YarnException e) { System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem."); } - LogCLIHelpers logCliHelper = new LogCLIHelpers(); - logCliHelper.setConf(getConf()); - - if (appOwner == null || appOwner.isEmpty()) { - appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); + if (getAMContainerLogs) { + if (logFiles == null || logFiles.length == 0) { + logFiles = new String[] { "syslog" }; + } + if (appState == YarnApplicationState.ACCEPTED + || appState == YarnApplicationState.RUNNING) { + return printAMContainerLogsForRunningApplication(getConf(), appIdStr, + amContainersList, logFiles, logCliHelper, appOwner); + } else { + if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { + return printAMContainerLogsForFinishedApplication(getConf(), + appIdStr, amContainersList, logFiles, logCliHelper, appOwner); + } else { + System.out + .println("Can not get AMContainers logs for the application:" + + appId); + System.out.println("This application:" + appId + " is finished." + + " Please enable the application history service. Or Using " + + "yarn logs -applicationId -containerId " + + "--nodeAddress to get the container logs"); + return -1; + } + } } + int resultCode = 0; - if (containerIdStr == null && nodeAddress == null) { - resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); - } else if ((containerIdStr == null && nodeAddress != null) - || (containerIdStr != null && nodeAddress == null)) { - System.out.println("ContainerId or NodeAddress cannot be null!"); - printHelpMessage(printOpts); - resultCode = -1; + if (containerIdStr != null) { + if (nodeAddress != null && isApplicationFinished(appState)) { + return logCliHelper.dumpAContainersLogs(appIdStr, containerIdStr, + nodeAddress, appOwner); + } + try { + ContainerReport report = getContainerReport(containerIdStr); + String nodeHttpAddress = + report.getNodeHttpAddress().replaceFirst( + WebAppUtils.getHttpSchemePrefix(getConf()), ""); + String nodeId = report.getAssignedNode().toString(); + if (!isApplicationFinished(appState)) { + if (logFiles == null || logFiles.length == 0) { + logFiles = new String[] { "syslog" }; + } + printContainerLogsFromRunningApplication(getConf(), appIdStr, + containerIdStr, nodeHttpAddress, nodeId, logFiles, logCliHelper, + appOwner); + } else { + printContainerLogsForFinishedApplication(appIdStr, containerIdStr, + nodeId, logFiles, logCliHelper, appOwner); + } + return resultCode; + } catch (IOException | YarnException ex) { + System.err.println("Unable to get logs for this container:" + + containerIdStr + "for the application:" + appId); + if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { + System.out.println("Please enable the application history service. Or "); + } + System.out.println("Using " + + "yarn logs -applicationId -containerId " + + "--nodeAddress to get the container logs"); + return -1; + } } else { - resultCode = - logCliHelper.dumpAContainersLogs(appIdStr, containerIdStr, - nodeAddress, appOwner); + if (nodeAddress == null) { + resultCode = + logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); + } else { + System.out.println("Should at least provide ContainerId!"); + printHelpMessage(printOpts); + resultCode = -1; + } } - return resultCode; } - private int verifyApplicationState(ApplicationId appId) throws IOException, - YarnException { + private YarnApplicationState getApplicationState(ApplicationId appId) + throws IOException, YarnException { YarnClient yarnClient = createYarnClient(); try { ApplicationReport appReport = yarnClient.getApplicationReport(appId); - switch (appReport.getYarnApplicationState()) { - case NEW: - case NEW_SAVING: - case SUBMITTED: - return -1; - case ACCEPTED: - case RUNNING: - case FAILED: - case FINISHED: - case KILLED: - default: - break; - - } + return appReport.getYarnApplicationState(); } finally { yarnClient.close(); } - return 0; } @VisibleForTesting @@ -201,4 +326,178 @@ private void printHelpMessage(Options options) { formatter.setSyntaxPrefix(""); formatter.printHelp("general options are:", options); } + + private JSONArray getAMContainerInfoForRunningApplication(Configuration conf, + String appId) throws ClientHandlerException, UniformInterfaceException, + JSONException { + Client webServiceClient = Client.create(); + String webAppAddress = + WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.RM_BIND_HOST, + WebAppUtils.getRMWebAppURLWithScheme(conf)); + WebResource webResource = webServiceClient.resource(webAppAddress); + + ClientResponse response = + webResource.path("ws").path("v1").path("cluster").path("apps") + .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + JSONObject json = + response.getEntity(JSONObject.class).getJSONObject("appAttempts"); + JSONArray requests = json.getJSONArray("appAttempt"); + return requests; + } + + private int printAMContainerLogsForRunningApplication(Configuration conf, + String appId, List amContainers, String[] logFiles, + LogCLIHelpers logCliHelper, String appOwner) throws Exception { + JSONArray amContainersInfo = null; + try { + amContainersInfo = getAMContainerInfoForRunningApplication(conf, appId); + } catch (Exception ex) { + System.err.println("Unable to get AM container informations " + + "for the application:" + appId); + System.err.println(ex); + return -1; + } + + boolean printAll = amContainers.contains("ALL"); + + for (int i = 0; i < amContainersInfo.length(); i++) { + boolean printThis = amContainers.contains(Integer.toString(i+1)) + || (i == (amContainersInfo.length()-1) + && amContainers.contains(Integer.toString(-1))); + if (printAll || printThis) { + String nodeHttpAddress = + amContainersInfo.getJSONObject(i).getString("nodeHttpAddress"); + String containerId = + amContainersInfo.getJSONObject(i).getString("containerId"); + String nodeId = amContainersInfo.getJSONObject(i).getString("nodeId"); + if (nodeHttpAddress != null && containerId != null + && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { + printContainerLogsFromRunningApplication(conf, appId, containerId, + nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner); + } + } + } + return 0; + } + + private JSONArray getAMContainerInfoForFinishedApplication(Configuration conf, + String appId) throws ClientHandlerException, UniformInterfaceException, + JSONException { + Client webServiceClient = Client.create(); + String webAppAddress = + WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + WebResource webResource = webServiceClient.resource(webAppAddress); + + ClientResponse response = + webResource.path("ws").path("v1").path("applicationhistory").path("apps") + .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + JSONObject json = response.getEntity(JSONObject.class); + JSONArray requests = json.getJSONArray("appAttempt"); + return requests; + } + + private int printAMContainerLogsForFinishedApplication(Configuration conf, + String appId, List amContainers, String[] logFiles, + LogCLIHelpers logCliHelper, String appOwner) throws Exception { + JSONArray amContainersInfo = null; + try { + amContainersInfo = + getAMContainerInfoForFinishedApplication(conf, appId); + } catch (Exception ex) { + System.err.println("Unable to get AM container informations " + + "for the application:" + appId); + System.err.println(ex); + return -1; + } + + boolean printAll = amContainers.contains("ALL"); + + for (int i = 0; i < amContainersInfo.length(); i++) { + boolean printThis = amContainers.contains(Integer.toString(i+1)) + || (i == (amContainersInfo.length()-1) + && amContainers.contains(Integer.toString(-1))); + if (printAll || printThis) { + String containerId = + amContainersInfo.getJSONObject(i).getString("amContainerId"); + if (containerId != null && !containerId.isEmpty()) { + String nodeAddress = null; + try { + nodeAddress = + getContainerReport(containerId).getAssignedNode().toString(); + } catch (Exception ex) { + System.err.println(ex); + nodeAddress = null; + } + if (nodeAddress != null && !nodeAddress.isEmpty()) { + printContainerLogsForFinishedApplication(appId, containerId, + nodeAddress, logFiles, logCliHelper, appOwner); + } + } + } + } + return 0; + } + + private void printContainerLogsFromRunningApplication(Configuration conf, + String appId, String containerIdStr, String nodeHttpAddress, + String nodeId, String[] logFiles, LogCLIHelpers logCliHelper, + String appOwner) throws IOException { + Client webServiceClient = Client.create(); + String containerString = "\n\nContainer: " + containerIdStr; + System.out.println(containerString); + System.out.println(StringUtils.repeat("=", containerString.length())); + for (String logFile : logFiles) { + System.out.println("LogType:" + logFile); + System.out.println("Log Upload Time:" + + Times.format(System.currentTimeMillis())); + System.out.println("Log Contents:"); + try { + WebResource webResource = + webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + + nodeHttpAddress); + ClientResponse response = + webResource.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(logFile) + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + System.out.println(response.getEntity(String.class)); + } catch (ClientHandlerException | UniformInterfaceException ex) { + System.out.println("Can not find the log file:" + logFile + + " for the container:" + containerIdStr + " in NodeManager:" + + nodeId); + } + } + // for the case, we have already uploaded partial logs in HDFS + logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId, + appOwner, Arrays.asList(logFiles)); + } + + private void printContainerLogsForFinishedApplication(String appId, + String containerId, String nodeAddress, String[] logFiles, + LogCLIHelpers logCliHelper, String appOwner) throws IOException { + String containerString = "\n\nContainer: " + containerId; + System.out.println(containerString); + System.out.println(StringUtils.repeat("=", containerString.length())); + logCliHelper.dumpAContainersLogsForALogType(appId, containerId, + nodeAddress, appOwner, logFiles != null ? Arrays.asList(logFiles) : null); + } + + private ContainerReport getContainerReport(String containerIdStr) + throws YarnException, IOException { + YarnClient yarnClient = createYarnClient(); + try { + return yarnClient.getContainerReport(ConverterUtils + .toContainerId(containerIdStr)); + } finally { + yarnClient.close(); + } + } + + private boolean isApplicationFinished(YarnApplicationState appState) { + return appState == YarnApplicationState.FINISHED + || appState == YarnApplicationState.FAILED + || appState == YarnApplicationState.KILLED; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 7ee918e..04f1017 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -145,14 +145,25 @@ public void testHelpMessage() throws Exception { pw.println("usage: yarn logs -applicationId [OPTIONS]"); pw.println(); pw.println("general options are:"); + pw.println(" -am Prints the AM Container logs for this"); + pw.println(" application. Specify comma-separated"); + pw.println(" value to get logs for related AM"); + pw.println(" Container. To get logs for all AM"); + pw.println(" Containers, use -am ALL. To get logs for"); + pw.println(" the latest AM Container, use -am -1. By"); + pw.println(" default, it will only print out syslog."); + pw.println(" Work with -logFiles to get other logs"); pw.println(" -appOwner AppOwner (assumed to be current user if"); pw.println(" not specified)"); - pw.println(" -containerId ContainerId (must be specified if node"); - pw.println(" address is specified)"); + pw.println(" -containerId ContainerId. By default, it will only"); + pw.println(" print syslog if the application is"); + pw.println(" runing. Work with -logFiles to get other"); + pw.println(" logs."); pw.println(" -help Displays help for all commands."); + pw.println(" -logFiles Work with -am/-containerId and specify"); + pw.println(" comma-separated value to get specified"); + pw.println(" Container log files"); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); - pw.println(" (must be specified if container id is"); - pw.println(" specified)"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 57f655b..debe770 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -745,6 +745,60 @@ public static void readAContainerLogsForALogType( readAContainerLogsForALogType(valueStream, out, -1); } + /** + * Keep calling this till you get a {@link EOFException} for getting logs of + * the specific types for a single container. + * @param valueStream + * @param out + * @param logUploadedTime + * @param logType + * @throws IOException + */ + public static int readContainerLogsForALogType( + DataInputStream valueStream, PrintStream out, long logUploadedTime, + List logType) throws IOException { + byte[] buf = new byte[65535]; + + String fileType = valueStream.readUTF(); + String fileLengthStr = valueStream.readUTF(); + long fileLength = Long.parseLong(fileLengthStr); + if (logType.contains(fileType)) { + out.print("LogType:"); + out.println(fileType); + if (logUploadedTime != -1) { + out.print("Log Upload Time:"); + out.println(Times.format(logUploadedTime)); + } + out.print("LogLength:"); + out.println(fileLengthStr); + out.println("Log Contents:"); + + long curRead = 0; + long pendingRead = fileLength - curRead; + int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; + int len = valueStream.read(buf, 0, toRead); + while (len != -1 && curRead < fileLength) { + out.write(buf, 0, len); + curRead += len; + + pendingRead = fileLength - curRead; + toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; + len = valueStream.read(buf, 0, toRead); + } + out.println("End of LogType:" + fileType); + out.println(""); + return 0; + } else { + long totalSkipped = 0; + long currSkipped = 0; + while (currSkipped != -1 && totalSkipped < fileLength) { + currSkipped = valueStream.skip(fileLength - totalSkipped); + totalSkipped += currSkipped; + } + return -1; + } + } + public void close() { IOUtils.cleanup(LOG, scanner, reader, fsDataIStream); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index df9bd32..39fd95e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -23,6 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; +import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -48,6 +49,14 @@ @VisibleForTesting public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { + return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner, + null); + } + + @Private + @VisibleForTesting + public int dumpAContainersLogsForALogType(String appId, String containerId, + String nodeId, String jobOwner, List logType) throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); @@ -78,9 +87,16 @@ public int dumpAContainersLogs(String appId, String containerId, reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); - if (dumpAContainerLogs(containerId, reader, System.out, + if (logType == null) { + if (dumpAContainerLogs(containerId, reader, System.out, thisNodeFile.getModificationTime()) > -1) { - foundContainerLogs = true; + foundContainerLogs = true; + } + } else { + if (dumpAContainerLogsForALogType(containerId, reader, System.out, + thisNodeFile.getModificationTime(), logType) > -1) { + foundContainerLogs = true; + } } } finally { if (reader != null) { @@ -131,6 +147,43 @@ public int dumpAContainerLogs(String containerIdStr, } @Private + public int dumpAContainerLogsForALogType(String containerIdStr, + AggregatedLogFormat.LogReader reader, PrintStream out, + long logUploadedTime, List logType) throws IOException { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + + while (valueStream != null && !key.toString().equals(containerIdStr)) { + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + + if (valueStream == null) { + return -1; + } + + boolean foundContainerLogs = false; + while (true) { + try { + int result = LogReader.readContainerLogsForALogType( + valueStream, out, logUploadedTime, logType); + if (result == 0) { + foundContainerLogs = true; + } + } catch (EOFException eof) { + break; + } + } + + if (foundContainerLogs) { + return 0; + } + return -1; + } + + @Private public int dumpAllContainersLogs(ApplicationId appId, String appOwner, PrintStream out) throws IOException { Path remoteRootLogDir = new Path(getConf().get(