diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a18ef7c..20c5c89 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -684,6 +684,14 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; /** + * Good local directories, use internally, + * initial value is the same as NM_LOCAL_DIRS. + */ + @Private + public static final String NM_GOOD_LOCAL_DIRS = + NM_PREFIX + "good-local-dirs"; + + /** * Number of files in each localized directories * Avoid tuning this too low. */ @@ -727,6 +735,14 @@ private static void addDeprecatedKeys() { public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; + /** + * Good log directories, use internally, + * initial value is the same as NM_LOG_DIRS. + */ + @Private + public static final String NM_GOOD_LOG_DIRS = + NM_PREFIX + "good-log-dirs"; + public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = NM_PREFIX + "resourcemanager.minimum.version"; public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 62ba599..c042f2c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -972,6 +972,12 @@ + Good local directories, use internally. + yarn.nodemanager.good-local-dirs + + + + It limits the maximum number of files which will be localized in a single local directory. If the limit is reached then sub-directories will be created and new files will be localized in them. If it is set to @@ -1043,6 +1049,12 @@ + Good log directories, use internally. + yarn.nodemanager.good-log-dirs + + + + Whether to enable log aggregation. Log aggregation collects each container's logs and moves these logs onto a file-system, for e.g. HDFS, after the application completes. Users can configure the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 6709c90..9aa3a23 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -113,9 +113,17 @@ public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException { new DirectoryCollection( validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)), maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB); + + String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS); + conf.set(YarnConfiguration.NM_GOOD_LOCAL_DIRS, + (local != null) ? local : ""); localDirsAllocator = new LocalDirAllocator( - YarnConfiguration.NM_LOCAL_DIRS); - logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS); + YarnConfiguration.NM_GOOD_LOCAL_DIRS); + String log = conf.get(YarnConfiguration.NM_LOG_DIRS); + conf.set(YarnConfiguration.NM_GOOD_LOG_DIRS, + (log != null) ? log : ""); + logDirsAllocator = new LocalDirAllocator( + YarnConfiguration.NM_GOOD_LOG_DIRS); } @Override @@ -373,10 +381,10 @@ private void updateDirsAfterTest() { Configuration conf = getConfig(); List localDirs = getLocalDirs(); - conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, + conf.setStrings(YarnConfiguration.NM_GOOD_LOCAL_DIRS, localDirs.toArray(new String[localDirs.size()])); List logDirs = getLogDirs(); - conf.setStrings(YarnConfiguration.NM_LOG_DIRS, + conf.setStrings(YarnConfiguration.NM_GOOD_LOG_DIRS, logDirs.toArray(new String[logDirs.size()])); if (!areDisksHealthy()) { // Just log. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java index c61d1f0..11f05e4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java @@ -120,6 +120,15 @@ public void testGetFullDirs() throws Exception { Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc()); Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals("", + dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOCAL_DIRS)); + Assert.assertEquals("", + dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOG_DIRS)); + Assert.assertEquals(localDir1 + "," + localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals(logDir1 + "," + logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f); nm = NodeManagerMetrics.create(); @@ -141,6 +150,15 @@ public void testGetFullDirs() throws Exception { Assert .assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals(localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOCAL_DIRS)); + Assert.assertEquals(logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_GOOD_LOG_DIRS)); + Assert.assertEquals(localDir1 + "," + localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals(logDir1 + "," + logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + FileUtils.deleteDirectory(new File(localDir1)); FileUtils.deleteDirectory(new File(localDir2)); FileUtils.deleteDirectory(new File(logDir1));