From 70ccf69567c2f430bbb874574ac8506f76687b22 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Wed, 10 Apr 2019 18:16:53 +0200 Subject: [PATCH] YARN-9471. Cleanup in TestLogAggregationIndexFileController --- ...tLogAggregationIndexedFileController.java} | 138 +++++++++--------- 1 file changed, 70 insertions(+), 68 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/{TestLogAggregationIndexFileController.java => TestLogAggregationIndexedFileController.java} (82%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java similarity index 82% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java index 79226797578a280a028c55e2df4558d92b0c697a..52ab5ba9b7bfa621698361fa1580a37caa7991ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java @@ -18,10 +18,6 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller.ifile; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileWriter; @@ -38,6 +34,7 @@ import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; @@ -63,15 +60,22 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ControlledClock; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + /** - * Function test for {@link LogAggregationIndexFileController}. + * Function test for {@link LogAggregationIndexedFileController}. * */ -public class TestLogAggregationIndexFileController { +public class TestLogAggregationIndexedFileController + extends Configured { private final String rootLocalLogDir = "target/LocalLogs"; private final Path rootLocalLogDirPath = new Path(rootLocalLogDir); @@ -81,37 +85,37 @@ private static final UserGroupInformation USER_UGI = UserGroupInformation .createRemoteUser("testUser"); private FileSystem fs; - private Configuration conf; private ApplicationId appId; private ContainerId containerId; private NodeId nodeId; private ByteArrayOutputStream sysOutStream; - private PrintStream sysOut; - private ByteArrayOutputStream sysErrStream; - private PrintStream sysErr; + private Configuration getTestConf() { + Configuration conf = new Configuration(); + conf.set("yarn.log-aggregation.Indexed.remote-app-log-dir", + remoteLogDir); + conf.set("yarn.log-aggregation.Indexed.remote-app-log-dir-suffix", + "logs"); + conf.set(YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, "gz"); + return conf; + } @Before public void setUp() throws IOException { + setConf(getTestConf()); appId = ApplicationId.newInstance(123456, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( appId, 1); containerId = ContainerId.newContainerId(attemptId, 1); nodeId = NodeId.newInstance("localhost", 9999); - conf = new Configuration(); - conf.set("yarn.log-aggregation.Indexed.remote-app-log-dir", - remoteLogDir); - conf.set("yarn.log-aggregation.Indexed.remote-app-log-dir-suffix", - "logs"); - conf.set(YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, "gz"); - fs = FileSystem.get(conf); + fs = FileSystem.get(getConf()); sysOutStream = new ByteArrayOutputStream(); - sysOut = new PrintStream(sysOutStream); + PrintStream sysOut = new PrintStream(sysOutStream); System.setOut(sysOut); - sysErrStream = new ByteArrayOutputStream(); - sysErr = new PrintStream(sysErrStream); + ByteArrayOutputStream sysErrStream = new ByteArrayOutputStream(); + PrintStream sysErr = new PrintStream(sysErrStream); System.setErr(sysErr); } @@ -172,7 +176,7 @@ public boolean isRollover(final FileContext fc, } }; - fileFormat.initialize(conf, "Indexed"); + fileFormat.initialize(getConf(), "Indexed"); Map appAcls = new HashMap<>(); Path appDir = fileFormat.getRemoteAppLogDir(appId, @@ -202,28 +206,28 @@ public boolean isRollover(final FileContext fc, logRequest.setBytes(Long.MAX_VALUE); List meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertTrue(meta.size() == 1); + assertEquals(1, meta.size()); List fileNames = new ArrayList<>(); for (ContainerLogMeta log : meta) { - Assert.assertTrue(log.getContainerId().equals(containerId.toString())); - Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); - Assert.assertTrue(log.getContainerLogMeta().size() == 3); + assertEquals(containerId.toString(), log.getContainerId()); + assertEquals(nodeId.toString(), log.getNodeId()); + assertEquals(3, log.getContainerLogMeta().size()); for (ContainerLogFileInfo file : log.getContainerLogMeta()) { fileNames.add(file.getFileName()); } } fileNames.removeAll(logTypes); - Assert.assertTrue(fileNames.isEmpty()); + assertTrue(fileNames.isEmpty()); boolean foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : logTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } sysOutStream.reset(); - Configuration factoryConf = new Configuration(conf); + Configuration factoryConf = new Configuration(getConf()); factoryConf.set("yarn.log-aggregation.file-formats", "Indexed"); factoryConf.set("yarn.log-aggregation.file-controller.Indexed.class", "org.apache.hadoop.yarn.logaggregation.filecontroller.ifile" @@ -232,12 +236,12 @@ public boolean isRollover(final FileContext fc, new LogAggregationFileControllerFactory(factoryConf); LogAggregationFileController fileController = factory .getFileControllerForRead(appId, USER_UGI.getShortUserName()); - Assert.assertTrue(fileController instanceof + assertTrue(fileController instanceof LogAggregationIndexedFileController); foundLogs = fileController.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : logTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } sysOutStream.reset(); @@ -260,12 +264,12 @@ public boolean isRollover(final FileContext fc, } meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertTrue(meta.size() == 0); + assertTrue(meta.isEmpty()); foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertFalse(foundLogs); + assertFalse(foundLogs); sysOutStream.reset(); fs.delete(checksumFile, false); - Assert.assertFalse(fs.exists(checksumFile)); + assertFalse(fs.exists(checksumFile)); List newLogTypes = new ArrayList<>(logTypes); files.clear(); @@ -288,26 +292,26 @@ public boolean isRollover(final FileContext fc, // We can only get the logs/logmeta from the first write. meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertEquals(meta.size(), 1); + assertEquals(1, meta.size()); for (ContainerLogMeta log : meta) { - Assert.assertTrue(log.getContainerId().equals(containerId.toString())); - Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); - Assert.assertTrue(log.getContainerLogMeta().size() == 3); + assertEquals(containerId.toString(), log.getContainerId()); + assertEquals(nodeId.toString(), log.getNodeId()); + assertEquals(3, log.getContainerLogMeta().size()); for (ContainerLogFileInfo file : log.getContainerLogMeta()) { fileNames.add(file.getFileName()); } } fileNames.removeAll(logTypes); - Assert.assertTrue(fileNames.isEmpty()); + assertTrue(fileNames.isEmpty()); foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : logTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } - Assert.assertFalse(sysOutStream.toString().contains(logMessage( + assertFalse(sysOutStream.toString().contains(logMessage( containerId, "test1"))); - Assert.assertFalse(sysOutStream.toString().contains(logMessage( + assertFalse(sysOutStream.toString().contains(logMessage( containerId, "test2"))); sysOutStream.reset(); @@ -319,20 +323,20 @@ public boolean isRollover(final FileContext fc, fileFormat.closeWriter(); meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertEquals(meta.size(), 2); + assertEquals(2, meta.size()); for (ContainerLogMeta log : meta) { - Assert.assertTrue(log.getContainerId().equals(containerId.toString())); - Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); + assertEquals(containerId.toString(), log.getContainerId()); + assertEquals(nodeId.toString(), log.getNodeId()); for (ContainerLogFileInfo file : log.getContainerLogMeta()) { fileNames.add(file.getFileName()); } } fileNames.removeAll(newLogTypes); - Assert.assertTrue(fileNames.isEmpty()); + assertTrue(fileNames.isEmpty()); foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : newLogTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } sysOutStream.reset(); @@ -344,23 +348,23 @@ public boolean isRollover(final FileContext fc, fileFormat.postWrite(context); fileFormat.closeWriter(); FileStatus[] status = fs.listStatus(logPath.getParent()); - Assert.assertTrue(status.length == 2); + assertEquals(2, status.length); meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertEquals(meta.size(), 3); + assertEquals(3, meta.size()); for (ContainerLogMeta log : meta) { - Assert.assertTrue(log.getContainerId().equals(containerId.toString())); - Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); + assertEquals(containerId.toString(), log.getContainerId()); + assertEquals(nodeId.toString(), log.getNodeId()); for (ContainerLogFileInfo file : log.getContainerLogMeta()) { fileNames.add(file.getFileName()); } } fileNames.removeAll(newLogTypes); - Assert.assertTrue(fileNames.isEmpty()); + assertTrue(fileNames.isEmpty()); foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : newLogTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } sysOutStream.reset(); @@ -389,7 +393,7 @@ public void testFetchApplictionLogsHar() throws Exception { assertTrue(fs.exists(harPath)); LogAggregationIndexedFileController fileFormat = new LogAggregationIndexedFileController(); - fileFormat.initialize(conf, "Indexed"); + fileFormat.initialize(getConf(), "Indexed"); ContainerLogsRequest logRequest = new ContainerLogsRequest(); logRequest.setAppId(appId); logRequest.setNodeId(nodeId.toString()); @@ -398,21 +402,21 @@ public void testFetchApplictionLogsHar() throws Exception { logRequest.setBytes(Long.MAX_VALUE); List meta = fileFormat.readAggregatedLogsMeta( logRequest); - Assert.assertEquals(meta.size(), 3); + assertEquals(3, meta.size()); List fileNames = new ArrayList<>(); for (ContainerLogMeta log : meta) { - Assert.assertTrue(log.getContainerId().equals(containerId.toString())); - Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); + assertEquals(containerId.toString(), log.getContainerId()); + assertEquals(nodeId.toString(), log.getNodeId()); for (ContainerLogFileInfo file : log.getContainerLogMeta()) { fileNames.add(file.getFileName()); } } fileNames.removeAll(newLogTypes); - Assert.assertTrue(fileNames.isEmpty()); + assertTrue(fileNames.isEmpty()); boolean foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); - Assert.assertTrue(foundLogs); + assertTrue(foundLogs); for (String logType : newLogTypes) { - Assert.assertTrue(sysOutStream.toString().contains(logMessage( + assertTrue(sysOutStream.toString().contains(logMessage( containerId, logType))); } sysOutStream.reset(); @@ -437,8 +441,6 @@ private File createAndWriteLocalLogFile(ContainerId containerId, } private String logMessage(ContainerId containerId, String logType) { - StringBuilder sb = new StringBuilder(); - sb.append("Hello " + containerId + " in " + logType + "!"); - return sb.toString(); + return "Hello " + containerId + " in " + logType + "!"; } } -- 2.21.0