From fcb2e261cea114068598afa6d0f3f17a4cfc2d39 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Fri, 21 Jun 2019 10:29:12 +0200 Subject: [PATCH] YARN-9629. Support configurable MIN_LOG_ROLLING_INTERVAL --- .../hadoop/yarn/conf/YarnConfiguration.java | 8 ++++ .../logaggregation/LogAggregationService.java | 47 ++++++++++++------- .../TestLogAggregationService.java | 37 +++++++++++++++ 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6bbcdcb1e11..9c1dab0c3ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1411,6 +1411,14 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; + /** + * The allowed hard minimum limit for {@link + * YarnConfiguration#NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS} + */ + public static final String MIN_LOG_ROLLING_INTERVAL = NM_PREFIX + + "log-aggregation.roll-monitoring-interval-seconds.min"; + public static final long MIN_LOG_ROLLING_INTERVAL_SUGGESTED = 3600; + /** * Define how many aggregated log files per application per NM we can have * in remote file system. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index d8db96780ff..56c372f7c57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -71,7 +71,6 @@ private static final Logger LOG = LoggerFactory.getLogger(LogAggregationService.class); - private static final long MIN_LOG_ROLLING_INTERVAL = 3600; // This configuration is for debug and test purpose. By setting // this configuration as true. We can break the lower bound of // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. @@ -116,29 +115,41 @@ protected void serviceInit(Configuration conf) throws Exception { rollingMonitorInterval = conf.getLong( YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); + long minRollingMonitorInterval = conf.getLong( + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL, + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SUGGESTED); boolean logAggregationDebugMode = conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, false); - if (rollingMonitorInterval > 0 - && rollingMonitorInterval < MIN_LOG_ROLLING_INTERVAL) { - if (logAggregationDebugMode) { - LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = " - + rollingMonitorInterval); - } else { - LOG.warn("rollingMonitorInterval should be more than or equal to {} " + - "seconds. Using {} seconds instead.", - MIN_LOG_ROLLING_INTERVAL, MIN_LOG_ROLLING_INTERVAL); - this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL; - } - } else if (rollingMonitorInterval <= 0) { + if (rollingMonitorInterval <= 0) { LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + ". The log rolling monitoring interval is disabled. " + "The logs will be aggregated after this application is finished."); } else { - LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval - + ". The logs will be aggregated every " + rollingMonitorInterval - + " seconds"); + if (!logAggregationDebugMode && minRollingMonitorInterval < + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SUGGESTED) { + LOG.warn("{} has been set to {}, which is less than the suggested " + + "minimum value {}. This may impact NodeManager's performance.", + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL, + minRollingMonitorInterval, + YarnConfiguration.MIN_LOG_ROLLING_INTERVAL_SUGGESTED); + } + if (rollingMonitorInterval < minRollingMonitorInterval) { + if (logAggregationDebugMode) { + LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = " + + rollingMonitorInterval); + } else { + LOG.warn("rollingMonitorInterval should be more than or equal to {} " + + "seconds. Using {} seconds instead.", + minRollingMonitorInterval, minRollingMonitorInterval); + this.rollingMonitorInterval = minRollingMonitorInterval; + } + } else { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The logs will be aggregated every " + rollingMonitorInterval + + " seconds"); + } } super.serviceInit(conf); @@ -413,6 +424,10 @@ public NodeId getNodeId() { return this.nodeId; } + @VisibleForTesting + public long getRollingMonitorInterval() { + return rollingMonitorInterval; + } private int getAggregatorThreadPoolSize(Configuration conf) { int threadPoolSize; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 484cad13f88..cf643e6a776 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2641,4 +2641,41 @@ public String getLogFilePathInLastCycle() { return this.logFileTypesInLastCycle; } } + + @Test + public void testRollingMonitorIntervalDefault() { + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(-1L, interval); + } + + @Test + public void testRollingMonitorIntervalGreaterThanSet() { + this.conf.set(YarnConfiguration.MIN_LOG_ROLLING_INTERVAL, "1800"); + this.conf.set(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, "2700"); + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(2700L, interval); + } + + @Test + public void testRollingMonitorIntervalLessThanSet() { + this.conf.set(YarnConfiguration.MIN_LOG_ROLLING_INTERVAL, "1800"); + this.conf.set(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, "600"); + LogAggregationService logAggregationService = + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler); + logAggregationService.init(this.conf); + + long interval = logAggregationService.getRollingMonitorInterval(); + assertEquals(1800L, interval); + } } -- 2.21.0