diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 7d20cf2..b83d625 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; @@ -32,6 +33,7 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.io.Writer; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -318,6 +320,89 @@ public void testFetchApplictionLogs() throws Exception { fs.delete(new Path(rootLogDir), true); } + @Test (timeout = 15000) + public void testFetchApplictionLogsHar() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + FileSystem fs = FileSystem.get(configuration); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + URL harUrl = ClassLoader.getSystemClassLoader() + .getResource("application_1440536969523_0001.har"); + assertNotNull(harUrl); + Path path = + new Path(remoteLogRootDir + ugi.getShortUserName() + + "/logs/application_1440536969523_0001"); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + Path harPath = new Path(path, "application_1440536969523_0001.har"); + fs.copyFromLocalFile(false, new Path(harUrl.toURI()), harPath); + assertTrue(fs.exists(harPath)); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + int exitCode = cli.run(new String[] { "-applicationId", + "application_1440536969523_0001"}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + "Container: container_1440536969523_0001_01_000001 on host1_1111\n" + + "=================================================================\n" + + "LogType:stderr\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:13\n" + + "Log Contents:\n" + + "Hello stderr\n" + + "End of LogType:stderr\n" + + "\n" + + "LogType:stdout\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:13\n" + + "Log Contents:\n" + + "Hello stdout\n" + + "End of LogType:stdout\n" + + "\n" + + "LogType:syslog\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:13\n" + + "Log Contents:\n" + + "Hello syslog\n" + + "End of LogType:syslog")); + assertTrue(sysOutStream.toString().contains( + "Container: container_1440536969523_0001_01_000002 on host2_2222\n" + + "=================================================================\n" + + "LogType:stderr\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:15\n" + + "Log Contents:\n" + + "Goodbye stderr\n" + + "End of LogType:stderr\n" + + "\n" + + "LogType:stdout\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:15\n" + + "Log Contents:\n" + + "Goodbye stdout\n" + + "End of LogType:stdout\n" + + "\n" + + "LogType:syslog\n" + + "Log Upload Time:Tue Aug 25 15:14:05 -0700 2015\n" + + "LogLength:15\n" + + "Log Contents:\n" + + "Goodbye syslog\n" + + "End of LogType:syslog\n")); + sysOutStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + } + private static void createContainerLogInLocalDir(Path appLogsDir, ContainerId containerId, FileSystem fs, List logTypes) throws Exception { Path containerLogsDir = new Path(appLogsDir, containerId.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_SUCCESS b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_SUCCESS new file mode 100644 index 0000000..e69de29 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index new file mode 100644 index 0000000..92ee728 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index @@ -0,0 +1,3 @@ +%2F dir 1440540845855+504+rkanter+supergroup 0 0 host1_1111 host2_2222 +%2Fhost1_1111 file part-0 0 394 1440540845834+420+rkanter+supergroup +%2Fhost2_2222 file part-0 394 400 1440540845854+420+rkanter+supergroup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex new file mode 100644 index 0000000..086d2b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex @@ -0,0 +1,2 @@ +3 +0 1520266628 0 214 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/part-0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/part-0 new file mode 100644 index 0000000000000000000000000000000000000000..fef262a81541f110716edd22e3443b2f121f8d1a GIT binary patch literal 795 zcmca8csXO@*6Z6W?>jn7+VwDilYu?VH7MBA-;afXfq_wwf!{G8z{k_sF$5?S@969! z%)k-m=^7pn6lP#A%1+EHNi7m)5JXnxAMWQG1T+XLtHGd_oS#>cn3{$_RZUL}V2%5~9duC?*NHV#pC5d1yXKt{u0fQ72s`4P$T|^6cbUwoAbdL$6K%|&3 wb nodeFiles; try { @@ -80,6 +82,11 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); + if (fileName.equals(applicationId + ".har")) { + Path p = new Path("har:///" + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null; @@ -207,6 +214,11 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, boolean foundAnyLogs = false; while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); + if (thisNodeFile.getPath().getName().equals(appId + ".har")) { + Path p = new Path("har:///" + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (!thisNodeFile.getPath().getName() .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index 620d097..e80063e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.security.UserGroupInformation; @@ -120,6 +121,11 @@ protected void render(Block html) { AggregatedLogFormat.LogReader reader = null; try { FileStatus thisNodeFile = nodeFiles.next(); + if (thisNodeFile.getPath().getName().equals(applicationId + ".har")) { + Path p = new Path("har:///" + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (!thisNodeFile.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) || thisNodeFile.getPath().getName() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java index 2a5762c..480dd17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java @@ -23,6 +23,7 @@ import java.io.FileWriter; import java.io.PrintWriter; import java.io.Writer; +import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -30,6 +31,7 @@ import javax.servlet.http.HttpServletRequest; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; @@ -144,8 +146,56 @@ public void testAggregatedLogsBlock() throws Exception { assertTrue(out.contains("test log1")); assertTrue(out.contains("test log2")); assertTrue(out.contains("test log3")); + } + + /** + * All ok and the AggregatedLogsBlockFor should aggregate logs and show it + * from a HAR archive. + * + * @throws Exception + */ + @Test + public void testAggregatedLogsBlockHar() throws Exception { + FileUtil.fullyDelete(new File("target/logs")); + Configuration configuration = getConfiguration(); + + URL harUrl = ClassLoader.getSystemClassLoader() + .getResource("application_1440536969523_0001.har"); + assertNotNull(harUrl); + String path = "target/logs/admin/logs/application_1440536969523_0001" + + "/application_1440536969523_0001.har"; + FileUtils.copyDirectory(new File(harUrl.getPath()), new File(path)); + + AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest( + configuration, "admin", + "container_1440536969523_0001_01_000001", "host1:1111"); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(data); + HtmlBlock html = new HtmlBlockForTest(); + HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false); + aggregatedBlock.render(block); + block.getWriter().flush(); + String out = data.toString(); + assertTrue(out.contains("Hello stderr")); + assertTrue(out.contains("Hello stdout")); + assertTrue(out.contains("Hello syslog")); + + aggregatedBlock = getAggregatedLogsBlockForTest( + configuration, "admin", + "container_1440536969523_0001_01_000002", "host2:2222"); + data = new ByteArrayOutputStream(); + printWriter = new PrintWriter(data); + html = new HtmlBlockForTest(); + block = new BlockForTest(html, printWriter, 10, false); + aggregatedBlock.render(block); + block.getWriter().flush(); + out = data.toString(); + assertTrue(out.contains("Goodbye stderr")); + assertTrue(out.contains("Goodbye stdout")); + assertTrue(out.contains("Goodbye syslog")); } + /** * Log files was deleted. * @throws Exception @@ -188,14 +238,20 @@ private Configuration getConfiguration() { private AggregatedLogsBlockForTest getAggregatedLogsBlockForTest( Configuration configuration, String user, String containerId) { + return getAggregatedLogsBlockForTest(configuration, user, containerId, + "localhost:1234"); + } + + private AggregatedLogsBlockForTest getAggregatedLogsBlockForTest( + Configuration configuration, String user, String containerId, + String nodeName) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getRemoteUser()).thenReturn(user); AggregatedLogsBlockForTest aggregatedBlock = new AggregatedLogsBlockForTest( configuration); aggregatedBlock.setRequest(request); aggregatedBlock.moreParams().put(YarnWebParams.CONTAINER_ID, containerId); - aggregatedBlock.moreParams().put(YarnWebParams.NM_NODENAME, - "localhost:1234"); + aggregatedBlock.moreParams().put(YarnWebParams.NM_NODENAME, nodeName); aggregatedBlock.moreParams().put(YarnWebParams.APP_OWNER, user); aggregatedBlock.moreParams().put("start", ""); aggregatedBlock.moreParams().put("end", ""); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_SUCCESS b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_SUCCESS new file mode 100644 index 0000000..e69de29 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index new file mode 100644 index 0000000..92ee728 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index @@ -0,0 +1,3 @@ +%2F dir 1440540845855+504+rkanter+supergroup 0 0 host1_1111 host2_2222 +%2Fhost1_1111 file part-0 0 394 1440540845834+420+rkanter+supergroup +%2Fhost2_2222 file part-0 394 400 1440540845854+420+rkanter+supergroup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex new file mode 100644 index 0000000..086d2b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex @@ -0,0 +1,2 @@ +3 +0 1520266628 0 214 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/part-0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/part-0 new file mode 100644 index 0000000000000000000000000000000000000000..fef262a81541f110716edd22e3443b2f121f8d1a GIT binary patch literal 795 zcmca8csXO@*6Z6W?>jn7+VwDilYu?VH7MBA-;afXfq_wwf!{G8z{k_sF$5?S@969! z%)k-m=^7pn6lP#A%1+EHNi7m)5JXnxAMWQG1T+XLtHGd_oS#>cn3{$_RZUL}V2%5~9duC?*NHV#pC5d1yXKt{u0fQ72s`4P$T|^6cbUwoAbdL$6K%|&3 wb