From 89b98b2e9c1505907f52f13094668724009a97ed Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Wed, 19 Jun 2019 16:22:39 +0200 Subject: [PATCH] YARN-9607. Auto-configuring rollover-size of IFile format for non-appendable filesystems --- .../LogAggregationFileController.java | 5 ---- .../LogAggregationIndexedFileController.java | 26 ++++++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 661e3219d33..43ae3c24a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -100,11 +100,6 @@ protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission .createImmutable((short) (0640 ^ 0777)); - // This is temporary solution. The configuration will be deleted once we have - // the FileSystem API to check whether append operation is supported or not. - public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND - = YarnConfiguration.YARN_PREFIX+ "log-aggregation.fs-support-append"; - protected Configuration conf; protected Path remoteRootLogDir; protected String remoteRootLogDirSuffix; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index a27d809904d..01eab007e26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import java.io.File; @@ -28,6 +29,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.net.URI; import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -109,6 +111,9 @@ private static final String LOG_ROLL_OVER_MAX_FILE_SIZE_GB = "indexedFile.log.roll-over.max-file-size-gb"; + private static final Set SUPPORTED_NON_APPENDABLE_FS_URIS = + ImmutableSet.of("s3a"); + @VisibleForTesting public static final String CHECK_SUM_FILE_SUFFIX = "-checksum"; @@ -135,16 +140,6 @@ public LogAggregationIndexedFileController() {} @Override public void initInternal(Configuration conf) { - // Currently, we need the underlying File System to support append - // operation. Will remove this check after we finish - // LogAggregationIndexedFileController for non-append mode. - boolean append = conf.getBoolean(LOG_AGGREGATION_FS_SUPPORT_APPEND, true); - if (!append) { - throw new YarnRuntimeException("The configuration:" - + LOG_AGGREGATION_FS_SUPPORT_APPEND + " is set as False. We can only" - + " use LogAggregationIndexedFileController when the FileSystem " - + "support append operations."); - } String compressName = conf.get( YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE); @@ -1139,6 +1134,17 @@ public static int getFSInputBufferSize(Configuration conf) { @Private @VisibleForTesting public long getRollOverLogMaxSize(Configuration conf) { + URI uri = remoteRootLogDir.toUri(); + if (uri != null) { + String scheme = uri.getScheme(); + if (scheme != null) { + if (SUPPORTED_NON_APPENDABLE_FS_URIS.contains(scheme.toLowerCase())) { + LOG.warn("Filesystem does not support append, rollover max size " + + "has been set to zero."); + return 0L; + } + } + } return 1024L * 1024 * 1024 * conf.getInt( LOG_ROLL_OVER_MAX_FILE_SIZE_GB, 10); } -- 2.21.0