diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 32b0934..ac43ecf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -95,11 +95,6 @@ // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED = YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled"; - private static final boolean - DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED = false; - - private static final long - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS = 3600; private final LocalDirsHandlerService dirsHandler; private final Dispatcher dispatcher; @@ -142,7 +137,7 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, Map appAcls, LogAggregationContext logAggregationContext, Context context, - FileContext lfs) { + FileContext lfs, long rollingMonitorInterval) { this.dispatcher = dispatcher; this.conf = conf; this.delService = deletionService; @@ -167,43 +162,7 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, } else { this.retentionSize = configuredRentionSize; } - long configuredRollingMonitorInterval = conf.getLong( - YarnConfiguration - .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, - YarnConfiguration - .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); - boolean debug_mode = - conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, - DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED); - if (configuredRollingMonitorInterval > 0 - && configuredRollingMonitorInterval < - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS) { - if (debug_mode) { - this.rollingMonitorInterval = configuredRollingMonitorInterval; - } else { - LOG.warn( - "rollingMonitorIntervall should be more than or equal to " - + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS - + " seconds. Using " - + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS - + " seconds instead."); - this.rollingMonitorInterval = - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS; - } - } else { - if (configuredRollingMonitorInterval <= 0) { - LOG.info("rollingMonitorInterval is set as " - + configuredRollingMonitorInterval + ". " - + "The log rolling monitoring interval is disabled. " - + "The logs will be aggregated after this application is finished."); - } else { - LOG.info("rollingMonitorInterval is set as " - + configuredRollingMonitorInterval + ". " - + "The logs will be aggregated every " - + configuredRollingMonitorInterval + " seconds"); - } - this.rollingMonitorInterval = configuredRollingMonitorInterval; - } + this.rollingMonitorInterval = rollingMonitorInterval; this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext == null || this.logAggregationContext.getRolledLogsIncludePattern() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 189e7ff..de03ebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -69,8 +69,14 @@ public class LogAggregationService extends AbstractService implements LogHandler { - private static final Log LOG = LogFactory - .getLog(LogAggregationService.class); + private static final Log LOG = LogFactory.getLog(LogAggregationService.class); + private static final long MIN_LOG_ROLLING_INTERVAL = 3600; + // This configuration is for debug and test purpose. By setting + // this configuration as true. We can break the lower bound of + // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. + private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED + = YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled"; + private long rollingMonitorInterval; /* * Expected deployment TLD will be 1777, owner=, group= 0 + && rollingMonitorInterval < MIN_LOG_ROLLING_INTERVAL) { + if (debug_mode) { + LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = " + + rollingMonitorInterval); + } else { + LOG.warn("rollingMonitorIntervall should be more than or equal to " + + MIN_LOG_ROLLING_INTERVAL + " seconds. Using " + + MIN_LOG_ROLLING_INTERVAL + " seconds instead."); + this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL; + } + } else if (rollingMonitorInterval <= 0) { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The log rolling monitoring interval is disabled. The logs will" + + " be aggregated after this application is finished."); + } else { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The logs will be aggregated every " + rollingMonitorInterval + + " seconds"); + } + super.serviceInit(conf); + verifyAndCreateRemoteLogDir(conf); } @Override @@ -321,7 +356,6 @@ private void initApp(final ApplicationId appId, String user, LogAggregationContext logAggregationContext) { ApplicationEvent eventResponse; try { - verifyAndCreateRemoteLogDir(getConfig()); initAppAggregator(appId, user, credentials, appAcls, logAggregationContext); eventResponse = new ApplicationEvent(appId, @@ -360,7 +394,7 @@ protected void initAppAggregator(final ApplicationId appId, String user, getConfig(), appId, userUgi, this.nodeId, dirsHandler, getRemoteNodeLogFileForApp(appId, user), appAcls, logAggregationContext, this.context, - getLocalFileContext(getConfig())); + getLocalFileContext(getConfig()), this.rollingMonitorInterval); if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) { throw new YarnRuntimeException("Duplicate initApp for " + appId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index fa9a0b5..7b44f61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -617,12 +617,10 @@ public void testVerifyAndCreateRemoteDirNonExistence() LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); - logAggregationService.init(this.conf); boolean existsBefore = aNewFile.exists(); assertTrue("The new file already exists!", !existsBefore); + logAggregationService.init(this.conf); - logAggregationService.verifyAndCreateRemoteLogDir(this.conf); - boolean existsAfter = aNewFile.exists(); assertTrue("The new aggregate file is not successfully created", existsAfter); aNewFile.delete(); //housekeeping