diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index da076eb..2705b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -799,6 +799,11 @@ private static void addDeprecatedKeys() { public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = NM_PREFIX + "remote-app-log-dir-suffix"; public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; + + /** Failover attempts associated with log aggregation dfs client */ + public static final String NM_REMOTE_APP_LOG_DFS_CLIENT_FAILOVER_MAX_ATTEMPTS = + NM_PREFIX + "remote-app-log-dfs-client-failover-max-attempts"; + public static final int DEFAULT_NM_REMOTE_APP_LOG_DFS_CLIENT_FAILOVER_MAX_ATTEMPTS = 2; public static final String YARN_LOG_SERVER_URL = YARN_PREFIX + "log.server.url"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 079d4d2..82addf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -883,6 +883,11 @@ yarn.nodemanager.remote-app-log-dir-suffix logs + + Failover attempts associated with log aggregation dfs client + yarn.nodemanager.remote-app-log-dfs-client-failover-max-attempts + 2 + Amount of physical memory, in MB, that can be allocated diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index bd3e847..a6680cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -127,6 +127,17 @@ protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); } + + @Override + protected void setConfig(Configuration conf) { + LOG.info("Cloning the config to inject specific configs for LogAggregation."); + int maxAttempts = conf.getInt(YarnConfiguration.NM_REMOTE_APP_LOG_DFS_CLIENT_FAILOVER_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DFS_CLIENT_FAILOVER_MAX_ATTEMPTS); + YarnConfiguration yarnConf = new YarnConfiguration(conf); + //TODO dfs.client.failover.max.attempts is hard coded now because this config is available in HDFS not in common. + yarnConf.setInt("dfs.client.failover.max.attempts", maxAttempts); + super.setConfig(yarnConf); + } @Override protected void serviceStart() throws Exception {