diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index d878d17..2323650 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -161,7 +161,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ -public final class FileContext { +public class FileContext { public static final Log LOG = LogFactory.getLog(FileContext.class); /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 23d2e72..395bc3d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -103,8 +103,8 @@ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); - // TODO: Why pick first app dir. The same in LCE why not random? - Path appStorageDir = getFirstApplicationDir(localDirs, user, appId); + // choose the local directory with maximum free space + Path appStorageDir = getWorkingDirectory(localDirs, user, appId); String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); @@ -461,9 +461,8 @@ public void deleteAsUser(String user, Path subDir, Path... baseDirs) * $logdir/$user/$appId */ static final short LOGDIR_PERM = (short)0710; - protected Path getFirstApplicationDir(List localDirs, String user, - String appId) { - return getApplicationDir(new Path(localDirs.get(0)), user, appId); + private long getDiskFreeSpace(Path base) throws IOException { + return lfs.getFsStatus(base).getRemaining(); } private Path getApplicationDir(Path base, String user, String appId) { @@ -493,6 +492,35 @@ protected void createDir(Path dirPath, FsPermission perms, } /** + * get working directory for a particular user. + */ + Path getWorkingDirectory(List localDirs, String user, String appId) + throws IOException { + Path appStorageDir = null; + long maxSpace = -1L; + // choose the app directory with maximum free space + for (String localDir : localDirs) { + Path curBase = getApplicationDir(new Path(localDir), + user, appId); + long space = 0L; + try { + space = getDiskFreeSpace(curBase); + } catch (IOException e) { + LOG.warn("Unable to get Free Space for " + curBase.toString(), e); + } + if (space > maxSpace) { + maxSpace = space; + appStorageDir = curBase; + } + } + if (appStorageDir == null) { + throw new IOException("Not able to find a working directory for " + + user); + } + return appStorageDir; + } + + /** * Initialize the local directories for a particular user. *