From e47ec267cdeb6a6781e336c10140c71583b7cda1 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Mon, 29 Apr 2019 15:35:48 +0200 Subject: [PATCH] YARN-9519. TFile log aggregation file format is insensitive to the yarn.log-aggregation.TFile.remote-app-log-dir config --- .../LogAggregationFileController.java | 43 +++++++++++++++++++ .../LogAggregationIndexedFileController.java | 21 +-------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 17a8aad59820509deea619f04f6b25d6b85e6008..bec6d64c2542fd422aba8b887f7896a0437e1c6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -132,6 +132,10 @@ public void initialize(Configuration conf, String controllerName) { this.retentionSize = configuredRetentionSize; } this.fileControllerName = controllerName; + + extractRemoteRootLogDir(); + extractRemoteRootLogDirSuffix(); + initInternal(conf); } @@ -248,6 +252,45 @@ public abstract String getApplicationOwner(Path aggregatedLogPath, public abstract Map getApplicationAcls( Path aggregatedLogPath, ApplicationId appId) throws IOException; + /** + * Sets the remoteRootLogDirSuffix class variable extracting + * {@link YarnConfiguration#LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT} + * from the configuration, or + * {@link YarnConfiguration#NM_REMOTE_APP_LOG_DIR_SUFFIX} appended by the + * FileController's name, if the former is not set. + */ + protected void extractRemoteRootLogDirSuffix() { + String suffix = String.format( + YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, + fileControllerName); + remoteRootLogDirSuffix = conf.get(suffix); + if (remoteRootLogDirSuffix == null + || remoteRootLogDirSuffix.isEmpty()) { + remoteRootLogDirSuffix = conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX) + + "-" + fileControllerName.toLowerCase(); + } + } + + /** + * Sets the remoteRootLogDir class variable extracting + * {@link YarnConfiguration#LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT} + * from the configuration or {@link YarnConfiguration#NM_REMOTE_APP_LOG_DIR}, + * if the former is not set. + */ + protected void extractRemoteRootLogDir() { + String remoteDirStr = String.format( + YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, + fileControllerName); + String remoteDir = conf.get(remoteDirStr); + if (remoteDir == null || remoteDir.isEmpty()) { + remoteDir = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); + } + remoteRootLogDir = new Path(remoteDir); + } + /** * Verify and create the remote log directory. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 9ab3e37bbdd1dddafbe7c8e23e91eb19f21839e9..e86bc49d6f9ef15de1eee263da6736df2ad97bd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -145,26 +145,7 @@ public void initInternal(Configuration conf) { + " use LogAggregationIndexedFileController when the FileSystem " + "support append operations."); } - String remoteDirStr = String.format( - YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, - this.fileControllerName); - String remoteDir = conf.get(remoteDirStr); - if (remoteDir == null || remoteDir.isEmpty()) { - remoteDir = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); - } - this.remoteRootLogDir = new Path(remoteDir); - String suffix = String.format( - YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, - this.fileControllerName); - this.remoteRootLogDirSuffix = conf.get(suffix); - if (this.remoteRootLogDirSuffix == null - || this.remoteRootLogDirSuffix.isEmpty()) { - this.remoteRootLogDirSuffix = conf.get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX) - + "-ifile"; - } + String compressName = conf.get( YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE); -- 2.21.0