diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2590b6f..5e08353 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1373,6 +1373,16 @@ public static boolean isAclEnabled(Configuration conf) { NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; public static final long DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; + + /** + * Define how many aggregated log files per application per NM we can have + * in remote file system. + */ + public static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP + = NM_PREFIX + "log-aggregation.num-log-files-per-app"; + public static final int + DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30; + /** * Number of threads used in log cleanup. Only applicable if Log aggregation * is disabled diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index aeef574..5ac89e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -95,13 +95,6 @@ protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission .createImmutable((short) (0640 ^ 0777)); - // This is temporary solution. The configuration will be deleted once - // we find a more scalable method to only write a single log file per LRS. - private static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP - = YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app"; - private static final int - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30; - // This is temporary solution. The configuration will be deleted once we have // the FileSystem API to check whether append operation is supported or not. public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND @@ -122,12 +115,14 @@ public LogAggregationFileController() {} */ public void initialize(Configuration conf, String controllerName) { this.conf = conf; - int configuredRentionSize = - conf.getInt(NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP, - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP); + int configuredRentionSize = conf.getInt( + YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP, + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP); if (configuredRentionSize <= 0) { this.retentionSize = - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP; + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP; } else { this.retentionSize = configuredRentionSize; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 81b6658..65ad200 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3012,6 +3012,15 @@ + Define how many aggregated log files per application per NM + we can have in remote file system. By default, the total number of + aggregated log files per application per NM is 30. + + yarn.nodemanager.log-aggregation.num-log-files-per-app + 30 + + + Enable/disable intermediate-data encryption at YARN level. For now, this only is used by the FileSystemRMStateStore to setup right