diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index d62ee5e..a8c6f1c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -412,7 +412,7 @@ public void printContainerLogsFromRunningApplication(Configuration conf, .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); out.println(response.getEntity(String.class)); - out.println("End of LogType:" + logFile); + out.println("\nEnd of LogType:" + logFile); out.flush(); } catch (ClientHandlerException | UniformInterfaceException ex) { System.err.println("Can not find the log file:" + logFile diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index db40b50..45b4541 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -377,7 +377,7 @@ public ContainerReport getContainerReport(String containerIdStr) String logMessage = "Hello container_0_0001_01_000003 in stdout!"; int fileContentSize = logMessage.getBytes().length; - int tailContentSize = "End of LogType:syslog\n\n".getBytes().length; + int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 98ffce1..8f660f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -776,6 +776,7 @@ private static void readContainerLogs(DataInputStream valueStream, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { @@ -790,6 +791,7 @@ private static void readContainerLogs(DataInputStream valueStream, } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -807,7 +809,9 @@ private static void readContainerLogs(DataInputStream valueStream, pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); } @@ -902,6 +906,7 @@ public static int readContainerLogsForALogType( long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { @@ -916,6 +921,7 @@ public static int readContainerLogsForALogType( } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -931,7 +937,9 @@ public static int readContainerLogsForALogType( toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); return 0; } else { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 692b172..22ef045 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -404,6 +404,7 @@ public void write(OutputStream os) throws IOException, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { @@ -418,6 +419,7 @@ public void write(OutputStream os) throws IOException, } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -435,6 +437,8 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); sb = new StringBuilder(); sb.append("\nEnd of LogType:" + fileType + "\n"); b = sb.toString().getBytes(Charset.forName("UTF-8")); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index efc0e7e..06aed65 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -264,6 +264,7 @@ public void write(OutputStream os) throws IOException, byte[] buf = new byte[bufferSize]; long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { @@ -278,6 +279,7 @@ public void write(OutputStream os) throws IOException, } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -295,6 +297,7 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = fis.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead); os.flush(); } finally { IOUtils.closeQuietly(fis); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 3961e1a..060042e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -948,7 +948,7 @@ private LogFileStatusInLastCycle verifyContainerLogs( Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size()); for (String fileType : logFiles) { String expectedValue = - containerStr + " Hello " + fileType + "!End of LogType:" + containerStr + " Hello " + fileType + "!\nEnd of LogType:" + fileType; LOG.info("Expected log-content : " + new String(expectedValue)); String foundValue = thisContainerMap.remove(fileType);