diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java index 32046c5..6a3a6f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java @@ -294,6 +294,7 @@ synchronized boolean checkDirs() { Set postCheckOtherDirs = new HashSet(errorDirs); for (String dir : preCheckFullDirs) { if (postCheckOtherDirs.contains(dir)) { + setChanged = true; LOG.warn("Directory " + dir + " error " + dirsFailedCheck.get(dir).message); } @@ -301,6 +302,7 @@ synchronized boolean checkDirs() { for (String dir : preCheckOtherErrorDirs) { if (postCheckFullDirs.contains(dir)) { + setChanged = true; LOG.warn("Directory " + dir + " error " + dirsFailedCheck.get(dir).message); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 0a61035..e81526c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.LocalDirAllocator; @@ -41,6 +42,8 @@ import org.apache.hadoop.yarn.server.nodemanager.DirectoryCollection.DirsChangeListener; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import com.google.common.annotations.VisibleForTesting; + /** * The class which provides functionality of checking the health of the local * directories of a node. This specifically manages nodemanager-local-dirs and @@ -50,6 +53,12 @@ private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class); + /** Use internally, initial value is the same as NM_LOG_DIRS. */ + @Private + @VisibleForTesting + static final String NM_LOG_DIRS_FOR_READ = + "yarn.nodemanager.log-dirs-for-read"; + /** Timer used to schedule disk health monitoring code execution */ private Timer dirsHandlerScheduler; private long diskHealthCheckInterval; @@ -81,6 +90,8 @@ */ private LocalDirAllocator logDirsAllocator; + private LocalDirAllocator logDirsForReadAllocator; + /** when disk health checking code was last run */ private long lastDisksCheckTime; @@ -114,6 +125,10 @@ public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException { localDirsAllocator = new LocalDirAllocator( YarnConfiguration.NM_LOCAL_DIRS); logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS); + + String log = conf.get(YarnConfiguration.NM_LOG_DIRS); + conf.set(NM_LOG_DIRS_FOR_READ, (log != null) ? log : ""); + logDirsForReadAllocator = new LocalDirAllocator(NM_LOG_DIRS_FOR_READ); } @Override @@ -376,6 +391,11 @@ private void updateDirsAfterTest() { List logDirs = getLogDirs(); conf.setStrings(YarnConfiguration.NM_LOG_DIRS, logDirs.toArray(new String[logDirs.size()])); + + List logDirsForRead = getLogDirsForRead(); + conf.setStrings(NM_LOG_DIRS_FOR_READ, + logDirsForRead.toArray(new String[logDirsForRead.size()])); + if (!areDisksHealthy()) { // Just log. LOG.error("Most of the disks failed. " + getDisksHealthReport(false)); @@ -484,9 +504,9 @@ public Path getLogPathForWrite(String pathStr, boolean checkWrite) } public Path getLogPathToRead(String pathStr) throws IOException { - return logDirsAllocator.getLocalPathToRead(pathStr, getConfig()); + return logDirsForReadAllocator.getLocalPathToRead(pathStr, getConfig()); } - + public static String[] validatePaths(String[] paths) { ArrayList validPaths = new ArrayList(); for (int i = 0; i < paths.length; ++i) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java index c61d1f0..d153ecf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java @@ -120,6 +120,13 @@ public void testGetFullDirs() throws Exception { Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc()); Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals("", + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals("", + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + Assert.assertEquals(logDir2, + dirSvc.getConfig().get(LocalDirsHandlerService.NM_LOG_DIRS_FOR_READ)); + conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f); nm = NodeManagerMetrics.create(); @@ -141,6 +148,13 @@ public void testGetFullDirs() throws Exception { Assert .assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals(localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals(logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + Assert.assertEquals(logDir2, + dirSvc.getConfig().get(LocalDirsHandlerService.NM_LOG_DIRS_FOR_READ)); + FileUtils.deleteDirectory(new File(localDir1)); FileUtils.deleteDirectory(new File(localDir2)); FileUtils.deleteDirectory(new File(logDir1)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java index e63f681..84e42fc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -39,6 +39,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.security.UserGroupInformation; @@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -144,6 +146,52 @@ public void testContainerLogDirs() throws IOException, YarnException { Assert.assertTrue(dirs.contains(containerLogDir)); } + @Test(timeout=30000) + public void testContainerLogFile() throws IOException, YarnException { + File absLogDir = new File("target", + TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile(); + String logdirwithFile = absLogDir.toURI().toString(); + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile); + conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, + 0.0f); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, + new ApplicationACLsManager(conf), new NMNullStateStoreService()); + // Add an application and the corresponding containers + String user = "nobody"; + long clusterTimeStamp = 1234; + ApplicationId appId = BuilderUtils.newApplicationId( + clusterTimeStamp, 1); + Application app = mock(Application.class); + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 1); + ContainerId containerId = BuilderUtils.newContainerId( + appAttemptId, 1); + nmContext.getApplications().put(appId, app); + + MockContainer container = + new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, + appId, 1); + container.setState(ContainerState.RUNNING); + nmContext.getContainers().put(containerId, container); + File containerLogDir = new File(absLogDir, + ContainerLaunch.getRelativeContainerLogDir(appId.toString(), + containerId.toString())); + containerLogDir.mkdirs(); + String fileName = "fileName"; + File containerLogFile = new File(containerLogDir, fileName); + containerLogFile.createNewFile(); + File file = ContainerLogsUtils.getContainerLogFile(containerId, + fileName, user, nmContext); + Assert.assertEquals(containerLogFile.toURI().toString(), + file.toURI().toString()); + FileUtil.fullyDelete(absLogDir); + } + @Test(timeout = 10000) public void testContainerLogPageAccess() throws IOException { // SecureIOUtils require Native IO to be enabled. This test will run