From 8333a1d386799681f9ecc72157f47123055a83f1 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Wed, 12 Jun 2019 16:23:05 +0200 Subject: [PATCH] YARN-9607. Auto-configuring rollover-size of IFile format for non-appendable filesystems --- .../LogAggregationIndexedFileController.java | 13 +++++ ...stLogAggregationIndexedFileController.java | 47 +++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index a27d809904d5fa5172901c4a2a223af2a2e6b241..0d15344ba0134da88e650befbd368bae648201ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -1139,6 +1139,14 @@ public static int getFSInputBufferSize(Configuration conf) { @Private @VisibleForTesting public long getRollOverLogMaxSize(Configuration conf) { + String scheme = remoteRootLogDir.toUri().getScheme(); + if (scheme != null) { + if (scheme.equalsIgnoreCase("s3a") || scheme.equalsIgnoreCase("s3n") || + scheme.equalsIgnoreCase("wasb") || scheme.equalsIgnoreCase("wasbs") || + scheme.equalsIgnoreCase("abfs") || scheme.equalsIgnoreCase("abfss")) { + return 0L; + } + } return 1024L * 1024 * 1024 * conf.getInt( LOG_ROLL_OVER_MAX_FILE_SIZE_GB, 10); } @@ -1236,4 +1244,9 @@ public Clock getSystemClock() { throw new IOException(ex); } } + + @VisibleForTesting + public long getLogRollOverMaxFileSize() { + return this.logRollOverMaxFileSize; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java index e63e469d982f4e1f08d242b3dae1d8f9253125c9..84c48ff60541730c7ed5363ccbb24b7597930ad0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java @@ -63,11 +63,9 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -444,4 +442,45 @@ private File createAndWriteLocalLogFile(ContainerId containerId, private String logMessage(ContainerId containerId, String logType) { return "Hello " + containerId + " in " + logType + "!"; } + + private void testControllerWithRemoteAppLogDir(String remoteAppLogDir, + boolean shouldBeZero) { + String testControllerName = "test"; + Configuration conf = new Configuration(); + conf.set(String.format(LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, + testControllerName), remoteAppLogDir); + + LogAggregationIndexedFileController fileController + = new LogAggregationIndexedFileController(); + fileController.initialize(conf, testControllerName); + if (shouldBeZero) { + assertEquals("TFile controller roll over max file size should be zero " + + "against cloud storage folders.", 0, + fileController.getLogRollOverMaxFileSize()); + } else { + assertNotEquals("TFile controller roll over max file size should not " + + "be zero against non cloud storage folders.", 0, + fileController.getLogRollOverMaxFileSize()); + } + } + + @Test + public void testRollOverWithS3AFolder() { + testControllerWithRemoteAppLogDir("s3a://log-bucket/folder/", true); + } + + @Test + public void testRollOverWithAzureFolder() { + testControllerWithRemoteAppLogDir("abfs://test/folder/", true); + } + + @Test + public void testRollOverWithHDFSFolder() { + testControllerWithRemoteAppLogDir("hdfs://dir/for/logs", false); + } + + @Test + public void testRollOverWithRelativePath() { + testControllerWithRemoteAppLogDir("/directory/log/", false); + } } -- 2.21.0