From 4e5872adbe0064a3b94cdd01d87e8ead95c876a8 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Fri, 21 Jun 2019 10:29:12 +0200 Subject: [PATCH] YARN-9629. Support configurable MIN_LOG_ROLLING_INTERVAL --- .../hadoop/yarn/conf/YarnConfiguration.java | 8 ++++ .../src/main/resources/yarn-default.xml | 15 +++++- .../logaggregation/LogAggregationService.java | 47 ++++++++++++------- .../TestLogAggregationService.java | 39 +++++++++++++++ 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6bbcdcb1e11..302ff1a6a64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1411,6 +1411,14 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; + /** + * The allowed hard minimum limit for {@link + * YarnConfiguration#NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS}. + */ + public static final String MIN_LOG_ROLLING_INTERVAL_SECONDS = NM_PREFIX + + "log-aggregation.roll-monitoring-interval-seconds.min"; + public static final long MIN_LOG_ROLLING_INTERVAL_SECONDS_SUGGESTED = 3600; + /** * Define how many aggregated log files per application per NM we can have * in remote file system. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9741f6c36b1..4e0e17a07c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3220,13 +3220,24 @@ Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded when the application is finished. By setting this configure, logs can be uploaded - periodically when the application is running. The minimum rolling-interval-seconds - can be set is 3600. + periodically when the application is running. + The minimum positive value is + "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min". yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds -1 + + Defines the hard positive minimum limit for + "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds". + If this configuration is set less than the default 3600 a NodeManager + may raise a warning. + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min + 3600 + + Define how many aggregated log files per application per NM we can have in remote file system. By default, the total number of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index d8db96780ff..c8a9386faa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -71,7 +71,6 @@ private static final Logger LOG = LoggerFactory.getLogger(LogAggregationService.class); - private static final long MIN_LOG_ROLLING_INTERVAL = 3600; // This configuration is for debug and test purpose. By setting // this configuration as true. We can break the lower bound of // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. @@ -116,29 +115,41 @@ protected void serviceInit(Configuration conf) throws Exception { rollingMonitorInterval = conf.getLong( YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); + long minRollingMonitorInterval = conf.getLong( + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS, + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS_SUGGESTED); boolean logAggregationDebugMode = conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, false); - if (rollingMonitorInterval > 0 - && rollingMonitorInterval < MIN_LOG_ROLLING_INTERVAL) { - if (logAggregationDebugMode) { - LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = " - + rollingMonitorInterval); - } else { - LOG.warn("rollingMonitorInterval should be more than or equal to {} " + - "seconds. Using {} seconds instead.", - MIN_LOG_ROLLING_INTERVAL, MIN_LOG_ROLLING_INTERVAL); - this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL; - } - } else if (rollingMonitorInterval <= 0) { + if (rollingMonitorInterval <= 0) { LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + ". The log rolling monitoring interval is disabled. " + "The logs will be aggregated after this application is finished."); } else { - LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval - + ". The logs will be aggregated every " + rollingMonitorInterval - + " seconds"); + if (!logAggregationDebugMode && minRollingMonitorInterval < + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS_SUGGESTED) { + LOG.warn("{} has been set to {}, which is less than the suggested " + + "minimum value {}. This may impact NodeManager's performance.", + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS, + minRollingMonitorInterval, + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS_SUGGESTED); + } + if (rollingMonitorInterval < minRollingMonitorInterval) { + if (logAggregationDebugMode) { + LOG.info("Log aggregation debug mode enabled. " + + "rollingMonitorInterval = " + rollingMonitorInterval); + } else { + LOG.warn("rollingMonitorInterval should be more than " + + "or equal to {} seconds. Using {} seconds instead.", + minRollingMonitorInterval, minRollingMonitorInterval); + this.rollingMonitorInterval = minRollingMonitorInterval; + } + } else { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The logs will be aggregated every " + rollingMonitorInterval + + " seconds"); + } } super.serviceInit(conf); @@ -413,6 +424,10 @@ public NodeId getNodeId() { return this.nodeId; } + @VisibleForTesting + public long getRollingMonitorInterval() { + return rollingMonitorInterval; + } private int getAggregatorThreadPoolSize(Configuration conf) { int threadPoolSize; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 484cad13f88..0e366827587 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2641,4 +2641,43 @@ public String getLogFilePathInLastCycle() { return this.logFileTypesInLastCycle; } } + + @Test + public void testRollingMonitorIntervalDefault() { + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(-1L, interval); + } + + @Test + public void testRollingMonitorIntervalGreaterThanSet() { + this.conf.set(YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS, "1800"); + this.conf.set(YarnConfiguration + .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, "2700"); + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(2700L, interval); + } + + @Test + public void testRollingMonitorIntervalLessThanSet() { + this.conf.set(YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SECONDS, "1800"); + this.conf.set(YarnConfiguration + .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, "600"); + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(1800L, interval); + } } -- 2.21.0