diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a18ef7c..20c5c89 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -684,6 +684,14 @@ private static void addDeprecatedKeys() {
public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
/**
+ * Good local directories, use internally,
+ * initial value is the same as NM_LOCAL_DIRS.
+ */
+ @Private
+ public static final String NM_GOOD_LOCAL_DIRS =
+ NM_PREFIX + "good-local-dirs";
+
+ /**
* Number of files in each localized directories
* Avoid tuning this too low.
*/
@@ -727,6 +735,14 @@ private static void addDeprecatedKeys() {
public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
+ /**
+ * Good log directories, use internally,
+ * initial value is the same as NM_LOG_DIRS.
+ */
+ @Private
+ public static final String NM_GOOD_LOG_DIRS =
+ NM_PREFIX + "good-log-dirs";
+
public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION =
NM_PREFIX + "resourcemanager.minimum.version";
public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 62ba599..c042f2c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -972,6 +972,12 @@
+ Good local directories, use internally.
+ yarn.nodemanager.good-local-dirs
+
+
+
+
It limits the maximum number of files which will be localized
in a single local directory. If the limit is reached then sub-directories
will be created and new files will be localized in them. If it is set to
@@ -1043,6 +1049,12 @@
+ Good log directories, use internally.
+ yarn.nodemanager.good-log-dirs
+
+
+
+
Whether to enable log aggregation. Log aggregation collects
each container's logs and moves these logs onto a file-system, for e.g.
HDFS, after the application completes. Users can configure the
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 6709c90..9aa3a23 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -113,9 +113,17 @@ public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException {
new DirectoryCollection(
validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)),
maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB);
+
+ String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS);
+ conf.set(YarnConfiguration.NM_GOOD_LOCAL_DIRS,
+ (local != null) ? local : "");
localDirsAllocator = new LocalDirAllocator(
- YarnConfiguration.NM_LOCAL_DIRS);
- logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
+ YarnConfiguration.NM_GOOD_LOCAL_DIRS);
+ String log = conf.get(YarnConfiguration.NM_LOG_DIRS);
+ conf.set(YarnConfiguration.NM_GOOD_LOG_DIRS,
+ (log != null) ? log : "");
+ logDirsAllocator = new LocalDirAllocator(
+ YarnConfiguration.NM_GOOD_LOG_DIRS);
}
@Override
@@ -373,10 +381,10 @@ private void updateDirsAfterTest() {
Configuration conf = getConfig();
List localDirs = getLocalDirs();
- conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS,
+ conf.setStrings(YarnConfiguration.NM_GOOD_LOCAL_DIRS,
localDirs.toArray(new String[localDirs.size()]));
List logDirs = getLogDirs();
- conf.setStrings(YarnConfiguration.NM_LOG_DIRS,
+ conf.setStrings(YarnConfiguration.NM_GOOD_LOG_DIRS,
logDirs.toArray(new String[logDirs.size()]));
if (!areDisksHealthy()) {
// Just log.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
index c61d1f0..11f05e4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
@@ -120,6 +120,15 @@ public void testGetFullDirs() throws Exception {
Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());
+ Assert.assertEquals("",
+ dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOCAL_DIRS));
+ Assert.assertEquals("",
+ dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOG_DIRS));
+ Assert.assertEquals(localDir1 + "," + localDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
+ Assert.assertEquals(logDir1 + "," + logDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
+
conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
100.0f);
nm = NodeManagerMetrics.create();
@@ -141,6 +150,15 @@ public void testGetFullDirs() throws Exception {
Assert
.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());
+ Assert.assertEquals(localDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOCAL_DIRS));
+ Assert.assertEquals(logDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOG_DIRS));
+ Assert.assertEquals(localDir1 + "," + localDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
+ Assert.assertEquals(logDir1 + "," + logDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
+
FileUtils.deleteDirectory(new File(localDir1));
FileUtils.deleteDirectory(new File(localDir2));
FileUtils.deleteDirectory(new File(logDir1));