diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 132dca2..2e9e92d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -227,6 +227,7 @@ public void testFetchApplictionLogs() throws Exception { assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001!")); + assertTrue(sysOutStream.toString().contains("LogUploadTime")); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index e1d1e00..185e260 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; @@ -215,6 +216,9 @@ public void write(DataOutputStream out, Set pendingUploadFiles) // Write the logFile Type out.writeUTF(logFile.getName()); + // Write the uploaded TimeStamp + out.writeLong(System.currentTimeMillis()); + // Write the log length as UTF so that it is printable out.writeUTF(String.valueOf(fileLength)); @@ -619,6 +623,7 @@ public static void readAcontainerLogs(DataInputStream valueStream, int bufferSize = 65536; char[] cbuf = new char[bufferSize]; String fileType; + long uploadTime; String fileLengthStr; long fileLength; @@ -629,10 +634,13 @@ public static void readAcontainerLogs(DataInputStream valueStream, // EndOfFile return; } + uploadTime = valueStream.readLong(); fileLengthStr = valueStream.readUTF(); fileLength = Long.parseLong(fileLengthStr); writer.write("\n\nLogType:"); writer.write(fileType); + writer.write("\nLogUploadTime:"); + writer.write(String.valueOf(uploadTime)); writer.write("\nLogLength:"); writer.write(fileLengthStr); writer.write("\nLog Contents:\n"); @@ -664,10 +672,13 @@ public static void readAContainerLogsForALogType( byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); + long uploadTime = valueStream.readLong(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); out.print("LogType: "); out.println(fileType); + out.print("LogUploadTime: "); + out.println(Times.format(uploadTime)); out.print("LogLength: "); out.println(fileLengthStr); out.println("Log Contents:"); @@ -698,6 +709,7 @@ public void close() { public static class ContainerLogsReader { private DataInputStream valueStream; private String currentLogType = null; + private long currentLogUpLoadTime = 0; private long currentLogLength = 0; private BoundedInputStream currentLogData = null; private InputStreamReader currentLogISR; @@ -718,12 +730,14 @@ public String nextLog() throws IOException { } currentLogType = null; + currentLogUpLoadTime = 0; currentLogLength = 0; currentLogData = null; currentLogISR = null; try { String logType = valueStream.readUTF(); + long logUpLoadTime = valueStream.readLong(); String logLengthStr = valueStream.readUTF(); currentLogLength = Long.parseLong(logLengthStr); currentLogData = @@ -731,6 +745,7 @@ public String nextLog() throws IOException { currentLogData.setPropagateClose(false); currentLogISR = new InputStreamReader(currentLogData); currentLogType = logType; + currentLogUpLoadTime = logUpLoadTime; } catch (EOFException e) { } @@ -741,6 +756,10 @@ public String getCurrentLogType() { return currentLogType; } + public long getCurrentLogUpLoadTime() { + return currentLogUpLoadTime; + } + public long getCurrentLogLength() { return currentLogLength; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index 16e6359..bba3258 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -198,12 +199,13 @@ private boolean readContainerLogs(Block html, if (desiredLogType == null || desiredLogType.isEmpty() || desiredLogType.equals(logType)) { long logLength = logReader.getCurrentLogLength(); - + long logUpLoadTime = logReader.getCurrentLogUpLoadTime(); if (foundLog) { html.pre()._("\n\n")._(); } html.p()._("Log Type: " + logType)._(); + html.p()._("Log UpLoadTime: " + Times.format(logUpLoadTime))._(); html.p()._("Log Length: " + Long.toString(logLength))._(); long start = logLimits.start < 0 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 676a156..8586303 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -37,7 +37,6 @@ import java.util.concurrent.CountDownLatch; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -57,6 +56,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter; +import org.apache.hadoop.yarn.util.Times; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -219,8 +219,11 @@ public void testReadAcontainerLogs1() throws Exception { LogReader.readAcontainerLogs(dis, writer); String s = writer.toString(); + int expectedLength = - "\n\nLogType:stdout".length() + ("\nLogLength:" + numChars).length() + "\n\nLogType:stdout".length() + + ("\nLogUploadTime:" + System.currentTimeMillis()) + .length() + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); @@ -232,7 +235,7 @@ public void testReadAcontainerLogs1() throws Exception { } String expectedContent = sb.toString(); Assert.assertTrue("Log content incorrect", s.contains(expectedContent)); - + Assert.assertEquals(expectedLength, s.length()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 8a5441a..cea71fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -61,7 +61,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -768,19 +767,23 @@ private String verifyContainerLogs(LogAggregationService logAggregationService, Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8)); String fileType = writtenLines[0].substring(9); - Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10)); - String fileLengthStr = writtenLines[1].substring(11); + Assert.assertEquals("LogUploadTime:", writtenLines[1].substring(0, 14)); + String fileUploadedTimeStr = writtenLines[1].substring(15); + + Assert.assertEquals("LogLength:", writtenLines[2].substring(0, 10)); + String fileLengthStr = writtenLines[2].substring(11); long fileLength = Long.parseLong(fileLengthStr); Assert.assertEquals("Log Contents:", - writtenLines[2].substring(0, 13)); + writtenLines[3].substring(0, 13)); String logContents = StringUtils.join( - Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n"); + Arrays.copyOfRange(writtenLines, 4, writtenLines.length), "\n"); perContainerMap.put(fileType, logContents); LOG.info("LogType:" + fileType); - LOG.info("LogType:" + fileLength); + LOG.info("LogUploadTime:" + fileUploadedTimeStr); + LOG.info("LogLength:" + fileLength); LOG.info("Log Contents:\n" + perContainerMap.get(fileType)); } catch (EOFException eof) { break;