diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index c3bdc0e..74c4846 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -77,6 +77,7 @@ private static final String SHOW_META_INFO = "show_meta_info"; private static final String LIST_NODES_OPTION = "list_nodes"; private static final String OUT_OPTION = "out"; + private static final String BYTES_OPTION = "bytes"; public static final String HELP_CMD = "help"; @Override @@ -102,6 +103,7 @@ public int run(String[] args) throws Exception { String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; + long bytes = Long.MAX_VALUE; try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); @@ -123,6 +125,14 @@ public int run(String[] args) throws Exception { if (commandLine.hasOption(CONTAINER_LOG_FILES)) { logFiles = commandLine.getOptionValues(CONTAINER_LOG_FILES); } + if (commandLine.hasOption(BYTES_OPTION)) { + bytes = Long.parseLong(commandLine.getOptionValue(BYTES_OPTION)); + if (bytes < 0) { + System.err.println("specified bytes should not be negative."); + printHelpMessage(printOpts); + return -1; + } + } } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); @@ -187,7 +197,7 @@ public int run(String[] args) throws Exception { LogsCLIOptions request = new LogsCLIOptions(appId, isApplicationFinished(appState), appOwner, - nodeAddress, null, containerIdStr, localDir, logs); + nodeAddress, null, containerIdStr, localDir, logs, bytes); if (showMetaInfo) { return showMetaInfo(request, logCliHelper); @@ -343,6 +353,7 @@ private void printContainerLogsFromRunningApplication(Configuration conf, ClientResponse response = webResource.path("ws").path("v1").path("node") .path("containerlogs").path(containerIdStr).path(logFile) + .queryParam("bytes", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); out.println(response.getEntity(String.class)); out.println("End of LogType:" + logFile); @@ -580,6 +591,7 @@ private Options createCommandOpts() { opts.addOption(OUT_OPTION, true, "Local directory for storing individual " + "container logs. The container logs will be stored based on the " + "node the container ran on."); + opts.addOption(BYTES_OPTION, true, "Prints the log file's last n bytes."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); @@ -600,6 +612,7 @@ private Options createPrintOpts(Options commandOpts) { printOpts.addOption(commandOpts.getOption(SHOW_META_INFO)); printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); + printOpts.addOption(commandOpts.getOption(BYTES_OPTION)); return printOpts; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aad0e19..893faef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -170,6 +170,7 @@ public void testHelpMessage() throws Exception { pw.println(" Work with -logFiles to get other logs"); pw.println(" -appOwner AppOwner (assumed to be current user if"); pw.println(" not specified)"); + pw.println(" -bytes Prints the log file's last n bytes."); pw.println(" -containerId ContainerId. By default, it will only"); pw.println(" print syslog if the application is"); pw.println(" runing. Work with -logFiles to get other"); @@ -300,6 +301,28 @@ public void testFetchApplictionLogs() throws Exception { "Hello container_0_0001_01_000003 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in stdout!")); + int fullSize = sysOutStream.toByteArray().length; + sysOutStream.reset(); + + // specify the bytes which is larger than the actual file size, + // we would get the full logs + exitCode = cli.run(new String[] { "-applicationId", appId.toString(), + "-logFiles", ".*", "-bytes", "10000" }); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toByteArray().length == fullSize); + sysOutStream.reset(); + + // specify how many bytes we should get from logs + exitCode = cli.run(new String[] { "-applicationId", appId.toString(), + "-logFiles", ".*", "-bytes", "5" }); + assertTrue(exitCode == 0); + int fileContentSize = + "Hello container_0_0001_01_000001 in syslog!".getBytes().length + + "Hello container_0_0001_01_000002 in syslog!".getBytes().length + + "Hello container_0_0001_01_000003 in syslog!".getBytes().length + + "Hello container_0_0001_01_000003 in stdout!".getBytes().length; + Assert.assertEquals(sysOutStream.toByteArray().length, + (fullSize - fileContentSize) + 20); sysOutStream.reset(); // uploaded two logs for container1. The first log is empty. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 15f5f87..9395e87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -667,7 +667,7 @@ public static void readAcontainerLogs(DataInputStream valueStream, ps = new PrintStream(os); while (true) { try { - readContainerLogs(valueStream, ps, logUploadedTime); + readContainerLogs(valueStream, ps, logUploadedTime, -1); } catch (EOFException e) { // EndOfFile return; @@ -691,7 +691,8 @@ public static void readAcontainerLogs(DataInputStream valueStream, } private static void readContainerLogs(DataInputStream valueStream, - PrintStream out, long logUploadedTime) throws IOException { + PrintStream out, long logUploadedTime, long bytes) + throws IOException { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); @@ -707,16 +708,24 @@ private static void readContainerLogs(DataInputStream valueStream, out.println(fileLengthStr); out.println("Log Contents:"); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength && bytes >= 0) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { out.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); @@ -736,8 +745,24 @@ private static void readContainerLogs(DataInputStream valueStream, */ public static void readAContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime) - throws IOException { - readContainerLogs(valueStream, out, logUploadedTime); + throws IOException { + readAContainerLogsForALogType(valueStream, out, logUploadedTime, -1); + } + + /** + * Keep calling this till you get a {@link EOFException} for getting logs of + * all types for a single container for the specific bytes. + * + * @param valueStream + * @param out + * @param logUploadedTime + * @param bytes + * @throws IOException + */ + public static void readAContainerLogsForALogType( + DataInputStream valueStream, PrintStream out, long logUploadedTime, + long bytes) throws IOException { + readContainerLogs(valueStream, out, logUploadedTime, bytes); } /** @@ -756,16 +781,33 @@ public static void readAContainerLogsForALogType( /** * Keep calling this till you get a {@link EOFException} for getting logs of + * a single container. + * @param valueStream + * @param out + * @param logUploadedTime + * @param logType + * @throws IOException + */ + public static int readContainerLogsForALogType( + DataInputStream valueStream, PrintStream out, long logUploadedTime, + List logType) throws IOException { + return readContainerLogsForALogType(valueStream, out, logUploadedTime, + logType, -1); + } + + /** + * Keep calling this till you get a {@link EOFException} for getting logs of * the specific types for a single container. * @param valueStream * @param out * @param logUploadedTime * @param logType + * @param bytes * @throws IOException */ public static int readContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime, - List logType) throws IOException { + List logType, long bytes) throws IOException { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); @@ -782,15 +824,23 @@ public static int readContainerLogsForALogType( out.println(fileLengthStr); out.println("Log Contents:"); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength && bytes >= 0) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { out.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 6a2c794..e93598e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -64,6 +64,7 @@ public int dumpAContainersLogs(String appId, String containerId, List logs = new ArrayList(); logs.add(".*"); options.setLogTypes(logs); + options.setBytes(-1); return dumpAContainersLogsForALogType(options, false); } @@ -157,7 +158,8 @@ public int dumpAContainersLogsForALogType(LogsCLIOptions options, new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { + thisNodeFile.getModificationTime(), logType, + options.getBytes()) > -1) { foundContainerLogs = true; } } finally { @@ -211,7 +213,8 @@ public int dumpAContainersLogsForALogTypeWithoutNodeId( out.println(containerId); out.println(StringUtils.repeat("=", containerId.length())); if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { + thisNodeFile.getModificationTime(), logType, + options.getBytes()) > -1) { foundContainerLogs = true; } } finally { @@ -229,33 +232,6 @@ public int dumpAContainersLogsForALogTypeWithoutNodeId( return 0; } - @Private - public int dumpAContainerLogs(String containerIdStr, - AggregatedLogFormat.LogReader reader, PrintStream out, - long logUploadedTime) throws IOException { - DataInputStream valueStream = getContainerLogsStream( - containerIdStr, reader); - - if (valueStream == null) { - return -1; - } - - boolean foundContainerLogs = false; - while (true) { - try { - LogReader.readAContainerLogsForALogType(valueStream, out, - logUploadedTime); - foundContainerLogs = true; - } catch (EOFException eof) { - break; - } - } - if (foundContainerLogs) { - return 0; - } - return -1; - } - private DataInputStream getContainerLogsStream(String containerIdStr, AggregatedLogFormat.LogReader reader) throws IOException { DataInputStream valueStream; @@ -273,7 +249,8 @@ private DataInputStream getContainerLogsStream(String containerIdStr, @Private public int dumpAContainerLogsForALogType(String containerIdStr, AggregatedLogFormat.LogReader reader, PrintStream out, - long logUploadedTime, List logType) throws IOException { + long logUploadedTime, List logType, long bytes) + throws IOException { DataInputStream valueStream = getContainerLogsStream( containerIdStr, reader); if (valueStream == null) { @@ -284,7 +261,7 @@ public int dumpAContainerLogsForALogType(String containerIdStr, while (true) { try { int result = LogReader.readContainerLogsForALogType( - valueStream, out, logUploadedTime, logType); + valueStream, out, logUploadedTime, logType, bytes); if (result == 0) { foundContainerLogs = true; } @@ -342,11 +319,13 @@ public int dumpAllContainersLogs(LogsCLIOptions options) throws IOException { try { if (logTypes == null) { LogReader.readAContainerLogsForALogType(valueStream, out, - thisNodeFile.getModificationTime()); + thisNodeFile.getModificationTime(), + options.getBytes()); foundAnyLogs = true; } else { - int result = LogReader.readContainerLogsForALogType(valueStream, - out, thisNodeFile.getModificationTime(), logTypes); + int result = LogReader.readContainerLogsForALogType( + valueStream, out, thisNodeFile.getModificationTime(), + logTypes, options.getBytes()); if (result == 0) { foundAnyLogs = true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java index 3f9a4f0..87383da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogsCLIOptions.java @@ -30,6 +30,7 @@ private boolean appFinished; private String outputLocalDir; private List LogTypes; + private long bytes; public LogsCLIOptions() {} @@ -42,12 +43,13 @@ public LogsCLIOptions(LogsCLIOptions request) { this.setContainerId(request.getContainerId()); this.setOutputLocalDir(request.getOutputLocalDir()); this.setLogTypes(request.getLogTypes()); + this.setBytes(request.getBytes()); } public LogsCLIOptions(ApplicationId applicationId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, - List logs) { + List logs, long bytes) { this.setAppId(applicationId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); @@ -56,6 +58,7 @@ public LogsCLIOptions(ApplicationId applicationId, this.setContainerId(container); this.setOutputLocalDir(localDir); this.setLogTypes(logs); + this.setBytes(bytes); } public ApplicationId getAppId() { @@ -121,4 +124,12 @@ public void setOutputLocalDir(String outputLocalDir) { public void setLogTypes(List logTypes) { LogTypes = logTypes; } + + public long getBytes() { + return bytes; + } + + public void setBytes(long bytes) { + this.bytes = bytes; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 502cebb..8aeaa55 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -23,9 +23,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Set; import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; @@ -215,7 +213,8 @@ public Response getLogs(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download) { + @QueryParam("download") String download, + @QueryParam("bytes") String bytes) { init(res); ContainerId containerId; try { @@ -226,7 +225,10 @@ public Response getLogs(@Context HttpServletRequest req, } boolean downloadFile = parseBooleanParam(download); - + final long size = parseLongParam(bytes); + if (size < 0) { + return Response.status(Status.BAD_REQUEST).build(); + } ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); AppInfo appInfo; @@ -235,7 +237,7 @@ public Response getLogs(@Context HttpServletRequest req, } catch (Exception ex) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, null, null, containerIdStr, - filename, downloadFile); + filename, downloadFile, size); } String appOwner = appInfo.getUser(); @@ -249,7 +251,7 @@ public Response getLogs(@Context HttpServletRequest req, if (isFinishedState(appInfo.getAppState())) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, downloadFile); + filename, downloadFile, size); } return createBadResponse(Status.INTERNAL_SERVER_ERROR, "Can not get ContainerInfo for the container: " + containerId); @@ -269,7 +271,7 @@ public Response getLogs(@Context HttpServletRequest req, return response.build(); } else if (isFinishedState(appInfo.getAppState())) { return sendStreamOutputResponse(appId, appOwner, nodeId, - containerIdStr, filename, downloadFile); + containerIdStr, filename, downloadFile, size); } else { return createBadResponse(Status.NOT_FOUND, "The application is not at Running or Finished State."); @@ -298,11 +300,11 @@ private boolean parseBooleanParam(String param) { private Response sendStreamOutputResponse(ApplicationId appId, String appOwner, String nodeId, String containerIdStr, - String fileName, boolean downloadFile) { + String fileName, boolean downloadFile, long bytes) { StreamingOutput stream = null; try { stream = getStreamingOutput(appId, appOwner, nodeId, - containerIdStr, fileName); + containerIdStr, fileName, bytes); } catch (Exception ex) { return createBadResponse(Status.INTERNAL_SERVER_ERROR, ex.getMessage()); @@ -322,7 +324,7 @@ private Response sendStreamOutputResponse(ApplicationId appId, private StreamingOutput getStreamingOutput(ApplicationId appId, String appOwner, final String nodeId, final String containerIdStr, - final String logFile) throws IOException{ + final String logFile, final long bytes) throws IOException{ String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path( conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -395,16 +397,24 @@ public void write(OutputStream os) throws IOException, byte[] b = sb.toString().getBytes(Charset.forName("UTF-8")); os.write(b, 0, b.length); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { os.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); @@ -442,4 +452,11 @@ private boolean isFileMatched(String filePattern, String fileName) { Pattern filterPattern = Pattern.compile(filePattern); return filterPattern.matcher(fileName).find(); } + + private long parseLongParam(String bytes) { + if (bytes == null || bytes.isEmpty()) { + return Long.MAX_VALUE; + } + return Long.parseLong(bytes); + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index e87b733..053fbbe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -654,6 +654,36 @@ public void testContainerLogsForFinishedApps() throws Exception { responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1ForApp100)); assertTrue(responseText.contains("Error." + containerId1ForApp100)); + int fullTextSize = responseText.getBytes().length; + + // specify how many bytes we should get from logs + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1ForApp100.toString()).path(".*") + .queryParam("user.name", user) + .queryParam("bytes", "5") + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + int fileContentSize = ("Hello." + containerId1ForApp100).getBytes().length + + ("Error." + containerId1ForApp100).getBytes().length; + assertEquals(responseText.getBytes().length, + (fullTextSize - fileContentSize) + 10); + assertTrue(fullTextSize >= responseText.getBytes().length); + + // specify the bytes which is larger than the actual file size, + // we would get the full logs + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1ForApp100.toString()).path(".*") + .queryParam("user.name", user) + .queryParam("bytes", "10000") + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(responseText.getBytes().length, fullTextSize); } private static void createContainerLogInLocalDir(Path appLogsDir, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 0a675f7..dece143 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -39,7 +39,6 @@ import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -220,7 +219,8 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context public Response getLogs(@javax.ws.rs.core.Context HttpServletRequest hsr, @PathParam("containerid") final String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download) throws IOException { + @QueryParam("download") String download, + @QueryParam("bytes") String bytes) throws IOException { ContainerId tempId; try { tempId = ConverterUtils.toContainerId(containerIdStr); @@ -238,6 +238,10 @@ public Response getLogs(@javax.ws.rs.core.Context } boolean downloadFile = parseBooleanParam(download); + final long size = parseLongParam(bytes); + if (size < 0) { + return Response.status(Status.BAD_REQUEST).build(); + } StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, @@ -254,6 +258,10 @@ public void write(OutputStream os) throws IOException, } FileInputStream fis = ContainerLogsUtils.openLogFileForRead( containerIdStr, logFile, nmContext); + long fileSize = logFile.length(); + if (fileSize > size) { + fis.skip(fileSize - size); + } int len; os.write(("\n" + file + "\n").getBytes()); while ((len = fis.read(buf, 0, bufferSize)) > 0) { @@ -290,4 +298,11 @@ private boolean parseBooleanParam(String param) { } return false; } + + private long parseLongParam(String bytes) { + if (bytes == null || bytes.isEmpty()) { + return Long.MAX_VALUE; + } + return Long.parseLong(bytes); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 860dda6..bd68f90 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -371,6 +371,27 @@ public void testContainerLogs() throws IOException { responseText = response.getEntity(String.class); Assert.assertTrue(responseText.contains(logMessage)); Assert.assertTrue(responseText.contains(errorMessage)); + int fullTextSize = responseText.getBytes().length; + + // specify how many bytes we should get from logs + response = r.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(".*") + .queryParam("bytes", "5") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + int expectedSize= ("\n" + filename + "\n").getBytes().length + + ("\n" + filename2 + "\n").getBytes().length + 10; + assertEquals(expectedSize, responseText.getBytes().length); + assertTrue(fullTextSize >= responseText.getBytes().length); + + // specify the bytes which is larger than the actual file size, + // we would get the full logs + response = r.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(".*") + .queryParam("bytes", "10000") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(fullTextSize, responseText.getBytes().length); // ask and download it response = r.path("ws").path("v1").path("node").path("containerlogs")