diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index b0e5f22..0ad1c29 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -86,6 +86,7 @@ private static final String SHOW_META_INFO = "show_meta_info"; private static final String LIST_NODES_OPTION = "list_nodes"; private static final String OUT_OPTION = "out"; + private static final String BYTES_OPTION = "bytes"; public static final String HELP_CMD = "help"; @Override @@ -111,6 +112,7 @@ public int run(String[] args) throws Exception { String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; + long bytes = Long.MAX_VALUE; try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); @@ -132,6 +134,14 @@ public int run(String[] args) throws Exception { if (commandLine.hasOption(CONTAINER_LOG_FILES)) { logFiles = commandLine.getOptionValues(CONTAINER_LOG_FILES); } + if (commandLine.hasOption(BYTES_OPTION)) { + bytes = Long.parseLong(commandLine.getOptionValue(BYTES_OPTION)); + if (bytes < 0) { + System.err.println("specified bytes should not be negative."); + printHelpMessage(printOpts); + return -1; + } + } } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); @@ -193,7 +203,7 @@ public int run(String[] args) throws Exception { ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, - containerIdStr, localDir, logs); + containerIdStr, localDir, logs, bytes); if (showMetaInfo) { return showMetaInfo(request, logCliHelper); @@ -398,6 +408,7 @@ private void printContainerLogsFromRunningApplication(Configuration conf, ClientResponse response = webResource.path("ws").path("v1").path("node") .path("containerlogs").path(containerIdStr).path(logFile) + .queryParam("bytes", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); out.println(response.getEntity(String.class)); out.println("End of LogType:" + logFile); @@ -632,6 +643,7 @@ private Options createCommandOpts() { opts.addOption(OUT_OPTION, true, "Local directory for storing individual " + "container logs. The container logs will be stored based on the " + "node the container ran on."); + opts.addOption(BYTES_OPTION, true, "Prints the log file's last n bytes."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); @@ -652,6 +664,7 @@ private Options createPrintOpts(Options commandOpts) { printOpts.addOption(commandOpts.getOption(SHOW_META_INFO)); printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); + printOpts.addOption(commandOpts.getOption(BYTES_OPTION)); return printOpts; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index c1af324..91ceaa0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -170,6 +170,7 @@ public void testHelpMessage() throws Exception { pw.println(" Work with -logFiles to get other logs"); pw.println(" -appOwner AppOwner (assumed to be current user if"); pw.println(" not specified)"); + pw.println(" -bytes Prints the log file's last n bytes."); pw.println(" -containerId ContainerId. By default, it will only"); pw.println(" print syslog if the application is"); pw.println(" runing. Work with -logFiles to get other"); @@ -300,6 +301,7 @@ public void testFetchApplictionLogs() throws Exception { "Hello container_0_0001_01_000003 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in stdout!")); + int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -322,6 +324,27 @@ public void testFetchApplictionLogs() throws Exception { "Can not find any log file matching the pattern: [123]")); sysErrStream.reset(); + // specify the bytes which is larger than the actual file size, + // we would get the full logs + exitCode = cli.run(new String[] { "-applicationId", appId.toString(), + "-logFiles", ".*", "-bytes", "10000" }); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toByteArray().length == fullSize); + sysOutStream.reset(); + + // specify how many bytes we should get from logs + exitCode = cli.run(new String[] { "-applicationId", appId.toString(), + "-logFiles", ".*", "-bytes", "5" }); + assertTrue(exitCode == 0); + int fileContentSize = + "Hello container_0_0001_01_000001 in syslog!".getBytes().length + + "Hello container_0_0001_01_000002 in syslog!".getBytes().length + + "Hello container_0_0001_01_000003 in syslog!".getBytes().length + + "Hello container_0_0001_01_000003 in stdout!".getBytes().length; + Assert.assertEquals(sysOutStream.toByteArray().length, + (fullSize - fileContentSize) + 20); + sysOutStream.reset(); + // uploaded two logs for container1. The first log is empty. // The second one is not empty. // We can still successfully read logs for container1. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index d636200..a090bde 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -733,7 +733,7 @@ public static void readAcontainerLogs(DataInputStream valueStream, ps = new PrintStream(os); while (true) { try { - readContainerLogs(valueStream, ps, logUploadedTime); + readContainerLogs(valueStream, ps, logUploadedTime, -1); } catch (EOFException e) { // EndOfFile return; @@ -757,7 +757,8 @@ public static void readAcontainerLogs(DataInputStream valueStream, } private static void readContainerLogs(DataInputStream valueStream, - PrintStream out, long logUploadedTime) throws IOException { + PrintStream out, long logUploadedTime, long bytes) + throws IOException { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); @@ -773,16 +774,24 @@ private static void readContainerLogs(DataInputStream valueStream, out.println(fileLengthStr); out.println("Log Contents:"); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength && bytes >= 0) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { out.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); @@ -803,7 +812,23 @@ private static void readContainerLogs(DataInputStream valueStream, public static void readAContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime) throws IOException { - readContainerLogs(valueStream, out, logUploadedTime); + readContainerLogs(valueStream, out, logUploadedTime, -1); + } + + /** + * Keep calling this till you get a {@link EOFException} for getting logs of + * all types for a single container for the specific bytes. + * + * @param valueStream + * @param out + * @param logUploadedTime + * @param bytes + * @throws IOException + */ + public static void readAContainerLogsForALogType( + DataInputStream valueStream, PrintStream out, long logUploadedTime, + long bytes) throws IOException { + readContainerLogs(valueStream, out, logUploadedTime, bytes); } /** @@ -832,6 +857,22 @@ public static void readAContainerLogsForALogType( public static int readContainerLogsForALogType( DataInputStream valueStream, PrintStream out, long logUploadedTime, List logType) throws IOException { + return readContainerLogsForALogType(valueStream, out, logUploadedTime, + logType, -1); + } + + /** + * Keep calling this till you get a {@link EOFException} for getting logs of + * the specific types for a single container. + * @param valueStream + * @param out + * @param logUploadedTime + * @param logType + * @throws IOException + */ + public static int readContainerLogsForALogType( + DataInputStream valueStream, PrintStream out, long logUploadedTime, + List logType, long bytes) throws IOException { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); @@ -848,15 +889,23 @@ public static int readContainerLogsForALogType( out.println(fileLengthStr); out.println("Log Contents:"); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength && bytes >= 0) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { out.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java index b0a7fdc..f32285c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java @@ -30,6 +30,7 @@ private boolean appFinished; private String outputLocalDir; private List logTypes; + private long bytes; public ContainerLogsRequest() {} @@ -42,12 +43,13 @@ public ContainerLogsRequest(ContainerLogsRequest request) { this.setContainerId(request.getContainerId()); this.setOutputLocalDir(request.getOutputLocalDir()); this.setLogTypes(request.getLogTypes()); + this.setBytes(request.getBytes()); } public ContainerLogsRequest(ApplicationId applicationId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, - List logs) { + List logs, long bytes) { this.setAppId(applicationId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); @@ -56,6 +58,7 @@ public ContainerLogsRequest(ApplicationId applicationId, this.setContainerId(container); this.setOutputLocalDir(localDir); this.setLogTypes(logs); + this.setBytes(bytes); } public ApplicationId getAppId() { @@ -121,4 +124,12 @@ public void setOutputLocalDir(String outputLocalDir) { public void setLogTypes(List logTypes) { this.logTypes = logTypes; } + + public long getBytes() { + return bytes; + } + + public void setBytes(long bytes) { + this.bytes = bytes; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 22147ae..3811054 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -65,6 +65,7 @@ public int dumpAContainersLogs(String appId, String containerId, options.setAppOwner(jobOwner); List logs = new ArrayList(); options.setLogTypes(logs); + options.setBytes(Long.MAX_VALUE); return dumpAContainersLogsForALogType(options, false); } @@ -160,12 +161,13 @@ public int dumpAContainersLogsForALogType(ContainerLogsRequest options, thisNodeFile.getPath()); if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, - thisNodeFile.getModificationTime()) > -1) { + thisNodeFile.getModificationTime(), options.getBytes()) > -1) { foundContainerLogs = true; } } else { if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { + thisNodeFile.getModificationTime(), logType, + options.getBytes()) > -1) { foundContainerLogs = true; } } @@ -222,12 +224,13 @@ public int dumpAContainersLogsForALogTypeWithoutNodeId( out.println(StringUtils.repeat("=", containerId.length())); if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, - thisNodeFile.getModificationTime()) > -1) { + thisNodeFile.getModificationTime(), options.getBytes()) > -1) { foundContainerLogs = true; } } else { if (dumpAContainerLogsForALogType(containerId, reader, out, - thisNodeFile.getModificationTime(), logType) > -1) { + thisNodeFile.getModificationTime(), logType, + options.getBytes()) > -1) { foundContainerLogs = true; } } @@ -249,7 +252,7 @@ public int dumpAContainersLogsForALogTypeWithoutNodeId( @Private public int dumpAContainerLogs(String containerIdStr, AggregatedLogFormat.LogReader reader, PrintStream out, - long logUploadedTime) throws IOException { + long logUploadedTime, long bytes) throws IOException { DataInputStream valueStream = getContainerLogsStream( containerIdStr, reader); @@ -261,7 +264,7 @@ public int dumpAContainerLogs(String containerIdStr, while (true) { try { LogReader.readAContainerLogsForALogType(valueStream, out, - logUploadedTime); + logUploadedTime, bytes); foundContainerLogs = true; } catch (EOFException eof) { break; @@ -290,7 +293,8 @@ private DataInputStream getContainerLogsStream(String containerIdStr, @Private public int dumpAContainerLogsForALogType(String containerIdStr, AggregatedLogFormat.LogReader reader, PrintStream out, - long logUploadedTime, List logType) throws IOException { + long logUploadedTime, List logType, long bytes) + throws IOException { DataInputStream valueStream = getContainerLogsStream( containerIdStr, reader); if (valueStream == null) { @@ -301,7 +305,7 @@ public int dumpAContainerLogsForALogType(String containerIdStr, while (true) { try { int result = LogReader.readContainerLogsForALogType( - valueStream, out, logUploadedTime, logType); + valueStream, out, logUploadedTime, logType, bytes); if (result == 0) { foundContainerLogs = true; } @@ -361,12 +365,13 @@ public int dumpAllContainersLogs(ContainerLogsRequest options) try { if (logTypes == null || logTypes.isEmpty()) { LogReader.readAContainerLogsForALogType(valueStream, out, - thisNodeFile.getModificationTime()); + thisNodeFile.getModificationTime(), + options.getBytes()); foundAnyLogs = true; } else { int result = LogReader.readContainerLogsForALogType( valueStream, out, thisNodeFile.getModificationTime(), - logTypes); + logTypes, options.getBytes()); if (result == 0) { foundAnyLogs = true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index e422c35..ed619f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -213,7 +213,8 @@ public Response getLogs(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download) { + @QueryParam("download") String download, + @QueryParam("bytes") String bytes) { init(res); ContainerId containerId; try { @@ -225,6 +226,11 @@ public Response getLogs(@Context HttpServletRequest req, boolean downloadFile = parseBooleanParam(download); + + final long size = parseLongParam(bytes); + if (size < 0) { + return Response.status(Status.BAD_REQUEST).build(); + } ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); AppInfo appInfo; @@ -233,7 +239,7 @@ public Response getLogs(@Context HttpServletRequest req, } catch (Exception ex) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, null, null, containerIdStr, - filename, downloadFile); + filename, downloadFile, size); } String appOwner = appInfo.getUser(); @@ -247,7 +253,7 @@ public Response getLogs(@Context HttpServletRequest req, if (isFinishedState(appInfo.getAppState())) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, downloadFile); + filename, downloadFile, size); } return createBadResponse(Status.INTERNAL_SERVER_ERROR, "Can not get ContainerInfo for the container: " + containerId); @@ -267,7 +273,7 @@ public Response getLogs(@Context HttpServletRequest req, return response.build(); } else if (isFinishedState(appInfo.getAppState())) { return sendStreamOutputResponse(appId, appOwner, nodeId, - containerIdStr, filename, downloadFile); + containerIdStr, filename, downloadFile, size); } else { return createBadResponse(Status.NOT_FOUND, "The application is not at Running or Finished State."); @@ -296,11 +302,11 @@ private boolean parseBooleanParam(String param) { private Response sendStreamOutputResponse(ApplicationId appId, String appOwner, String nodeId, String containerIdStr, - String fileName, boolean downloadFile) { + String fileName, boolean downloadFile, long bytes) { StreamingOutput stream = null; try { stream = getStreamingOutput(appId, appOwner, nodeId, - containerIdStr, fileName); + containerIdStr, fileName, bytes); } catch (Exception ex) { return createBadResponse(Status.INTERNAL_SERVER_ERROR, ex.getMessage()); @@ -318,7 +324,7 @@ private Response sendStreamOutputResponse(ApplicationId appId, private StreamingOutput getStreamingOutput(ApplicationId appId, String appOwner, final String nodeId, final String containerIdStr, - final String logFile) throws IOException{ + final String logFile, final long bytes) throws IOException{ String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path( conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -391,16 +397,24 @@ public void write(OutputStream os) throws IOException, byte[] b = sb.toString().getBytes(Charset.forName("UTF-8")); os.write(b, 0, b.length); + long toSkip = 0; + long actualRead = fileLength; + if (bytes < fileLength) { + toSkip = fileLength - bytes; + actualRead = bytes; + } + valueStream.skip(toSkip); + long curRead = 0; - long pendingRead = fileLength - curRead; + long pendingRead = actualRead - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { + while (len != -1 && curRead < actualRead) { os.write(buf, 0, len); curRead += len; - pendingRead = fileLength - curRead; + pendingRead = actualRead - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); @@ -433,4 +447,11 @@ public void write(OutputStream os) throws IOException, }; return stream; } + + private long parseLongParam(String bytes) { + if (bytes == null || bytes.isEmpty()) { + return Long.MAX_VALUE; + } + return Long.parseLong(bytes); + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index f985fe4..7f86a1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -601,6 +601,35 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1ForApp100)); + int fullTextSize = responseText.getBytes().length; + + // specify how many bytes we should get from logs + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1ForApp100.toString()).path(fileName) + .queryParam("user.name", user) + .queryParam("bytes", "5") + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + int fileContentSize = ("Hello." + containerId1ForApp100).getBytes().length; + assertEquals(responseText.getBytes().length, + (fullTextSize - fileContentSize) + 5); + assertTrue(fullTextSize >= responseText.getBytes().length); + + // specify the bytes which is larger than the actual file size, + // we would get the full logs + r = resource(); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containerlogs") + .path(containerId1ForApp100.toString()).path(fileName) + .queryParam("user.name", user) + .queryParam("bytes", "10000") + .accept(MediaType.TEXT_PLAIN) + .get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(responseText.getBytes().length, fullTextSize); } private static void createContainerLogInLocalDir(Path appLogsDir, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 5c66511..ffaabaa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -217,7 +217,8 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context @Unstable public Response getLogs(@PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download) { + @QueryParam("download") String download, + @QueryParam("bytes") String bytes) { ContainerId containerId; try { containerId = ConverterUtils.toContainerId(containerIdStr); @@ -235,10 +236,19 @@ public Response getLogs(@PathParam("containerid") String containerIdStr, return Response.serverError().entity(ex.getMessage()).build(); } boolean downloadFile = parseBooleanParam(download); + final long size = parseLongParam(bytes); + if (size < 0) { + return Response.status(Status.BAD_REQUEST).build(); + } + try { final FileInputStream fis = ContainerLogsUtils.openLogFileForRead( containerIdStr, logFile, nmContext); - + long fileSize = logFile.length(); + if (fileSize > size) { + fis.skip(fileSize - size); + } + StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, @@ -268,4 +278,11 @@ private boolean parseBooleanParam(String param) { } return false; } + + private long parseLongParam(String bytes) { + if (bytes == null || bytes.isEmpty()) { + return Long.MAX_VALUE; + } + return Long.parseLong(bytes); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index ce1b309..832e4ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -36,7 +36,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.NodeHealthScriptRunner; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -351,6 +350,25 @@ public void testContainerLogs() throws IOException { .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); String responseText = response.getEntity(String.class); assertEquals(logMessage, responseText); + int fullTextSize = responseText.getBytes().length; + + // specify how many bytes we should get from logs + response = r.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(filename) + .queryParam("bytes", "5") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(5, responseText.getBytes().length); + assertTrue(fullTextSize >= responseText.getBytes().length); + + // specify the bytes which is larger than the actual file size, + // we would get the full logs + response = r.path("ws").path("v1").path("node") + .path("containerlogs").path(containerIdStr).path(filename) + .queryParam("bytes", "10000") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals(fullTextSize, responseText.getBytes().length); // ask and download it response = r.path("ws").path("v1").path("node").path("containerlogs")