diff --git conf/taskcontroller.cfg conf/taskcontroller.cfg index fb4cced..eb2e5ee 100644 --- conf/taskcontroller.cfg +++ conf/taskcontroller.cfg @@ -1,3 +1,2 @@ -mapred.local.dir=#configured value of hadoop.tmp.dir it can be a list of paths comma seperated -hadoop.pid.dir=#configured HADOOP_PID_DIR -hadoop.indent.str=#configured HADOOP_IDENT_STR +mapred.local.dir=#configured value of mapred.local.dir. It can be a list of comma separated paths. +hadoop.log.dir=#configured value of hadoop.log.dir. \ No newline at end of file diff --git src/java/org/apache/hadoop/fs/FileUtil.java src/java/org/apache/hadoop/fs/FileUtil.java index d1b1d0b..62966cc 100644 --- src/java/org/apache/hadoop/fs/FileUtil.java +++ src/java/org/apache/hadoop/fs/FileUtil.java @@ -22,17 +22,24 @@ import java.io.*; import java.util.Enumeration; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; + +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; -import org.mortbay.log.Log; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * A collection of file-processing util methods */ public class FileUtil { + + final static Log LOG = LogFactory.getLog(FileUtil.class); + /** * convert an array of FileStatus to an array of Path * @@ -679,6 +686,43 @@ public class FileUtil { } /** + * Permission information useful for setting permissions for a given path. + * Using this, one can set all possible combinations of permissions for the + * owner of the file. But permissions for the group and all others can only be + * set together, i.e. permissions for group cannot be set different from those + * for others and vice versa. + */ + public static class PermissionsInfo { + public boolean readPermissions; + public boolean writePermissions; + public boolean executablePermissions; + public boolean readPermsOwnerOnly; + public boolean writePermsOwnerOnly; + public boolean executePermsOwnerOnly; + + /** + * Create a permissions-info object with the given attributes + * + * @param readPerms + * @param writePerms + * @param executePerms + * @param readOwnerOnly + * @param writeOwnerOnly + * @param executeOwnerOnly + */ + public PermissionsInfo(boolean readPerms, boolean writePerms, + boolean executePerms, boolean readOwnerOnly, boolean writeOwnerOnly, + boolean executeOwnerOnly) { + readPermissions = readPerms; + writePermissions = writePerms; + executablePermissions = executePerms; + readPermsOwnerOnly = readOwnerOnly; + writePermsOwnerOnly = writeOwnerOnly; + executePermsOwnerOnly = executeOwnerOnly; + } + } + + /** * Create a soft link between a src and destination * only on a local disk. HDFS does not support this * @param target the target for symlink @@ -734,8 +778,8 @@ public class FileUtil { try { shExec.execute(); }catch(Exception e) { - if(Log.isDebugEnabled()) { - Log.debug("Error while changing permission : " + filename + if(LOG.isDebugEnabled()) { + LOG.debug("Error while changing permission : " + filename +" Exception: " + StringUtils.stringifyException(e)); } } @@ -791,4 +835,64 @@ public class FileUtil { } } } + + /** + * Set permission on the given file path using the specified permissions + * information. We use java api to set permission instead of spawning chmod + * processes. This saves a lot of time. Using this, one can set all possible + * combinations of permissions for the owner of the file. But permissions for + * the group and all others can only be set together, i.e. permissions for + * group cannot be set different from those for others and vice versa. + * + * This method should satisfy the needs of most of the applications. For those + * it doesn't, {@link chmod} can be used. + * + * @param f file path + * @param pInfo permissions information + * @return true if success, false otherwise + */ + public static boolean setPermissions(File f, PermissionsInfo pInfo) { + if (pInfo == null) { + LOG.debug(" PermissionsInfo is null, returning."); + return true; + } + + LOG.debug("Setting permission for " + f.getAbsolutePath()); + + // Clear all the flags + f.setReadable(false, false); + f.setWritable(false, false); + f.setExecutable(false, false); + + boolean ret = true; + ret = f.setReadable(pInfo.readPermissions, pInfo.readPermsOwnerOnly); + LOG.debug("Readable status for " + f + " set to " + ret); + ret = + f.setWritable(pInfo.writePermissions, pInfo.writePermsOwnerOnly) + && ret; + LOG.debug("Writable status for " + f + " set to " + ret); + ret = + f.setExecutable(pInfo.executablePermissions, + pInfo.executePermsOwnerOnly) + && ret; + + LOG.debug("Executable status for " + f + " set to " + ret); + return ret; + } + + /** + * Permissions rwxr_xr_x + */ + public static PermissionsInfo sevenFiveFive = + new PermissionsInfo(true, true, true, false, true, false); + /** + * Completely private permissions + */ + public static PermissionsInfo sevenZeroZero = + new PermissionsInfo(true, true, true, true, true, true); + /** + * Completely open permissions + */ + public static PermissionsInfo sevenSevenSeven = + new PermissionsInfo(true, true, true, false, false, false); } diff --git src/java/org/apache/hadoop/util/DiskChecker.java src/java/org/apache/hadoop/util/DiskChecker.java index 4c471db..c3cb74b 100644 --- src/java/org/apache/hadoop/util/DiskChecker.java +++ src/java/org/apache/hadoop/util/DiskChecker.java @@ -21,12 +21,19 @@ package org.apache.hadoop.util; import java.io.File; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; + /** * Class that provides utility functions for checking disk problem */ public class DiskChecker { + final static Log LOG = + LogFactory.getLog(DiskChecker.class); + public static class DiskErrorException extends IOException { public DiskErrorException(String msg) { super(msg); @@ -53,37 +60,105 @@ public class DiskChecker { * @return true on success, false on failure */ public static boolean mkdirsWithExistsCheck(File dir) { - if (dir.mkdir() || dir.exists()) { + return mkdirsWithExistsCheck(dir, null); + } + + /** + * The semantics of mkdirsWithExistsCheck method is different from the mkdirs + * method provided in the Sun's java.io.File class in the following way: While + * creating the non-existent parent directories, this method checks for the + * existence of those directories if the mkdir fails at any point (since that + * directory might have just been created by some other process). If both + * mkdir() and the exists() check fails for any seemingly non-existent + * directory, then we signal an error; Sun's mkdir would signal an error + * (return false) if a directory it is attempting to create already exists or + * the mkdir fails. + * + * Using the permissions information passed, the leaf of the newly + * created directories is set appropriate permissions. + * + * @param dir + * @param dirCreationPermisInfo + * @return true on success, false on failure + */ + public static boolean mkdirsWithExistsCheck(File dir, + FileUtil.PermissionsInfo dirCreationPermisInfo) { + if (dir.exists()) { + LOG.debug(dir + " exists, not setting permissions. Returning"); return true; + } else if (dir.mkdir()) { + LOG.debug(dir + " creating, setting permissions and returning"); + return FileUtil.setPermissions(dir, dirCreationPermisInfo); } + File canonDir = null; try { canonDir = dir.getCanonicalFile(); } catch (IOException e) { return false; } + String parent = canonDir.getParent(); - return (parent != null) && - (mkdirsWithExistsCheck(new File(parent)) && - (canonDir.mkdir() || canonDir.exists())); + + boolean parentCreation = + (parent != null) + && mkdirsWithExistsCheck(new File(parent), dirCreationPermisInfo); + LOG.debug("Status for creation of the parent " + parent + " : " + " " + + parentCreation); + boolean canonDirCreation = + parentCreation && (canonDir.mkdir() || canonDir.exists()); + LOG.debug("Status for creation of the directory " + canonDir + " : " + " " + + canonDirCreation); + boolean canonDirPermissions = + canonDirCreation && FileUtil.setPermissions(canonDir, dirCreationPermisInfo); + LOG.debug("Status for setting permissions for the directory " + canonDir + + " " + canonDirPermissions); + return canonDirPermissions; } - + + /** + * Tries creating the given directory, and if needed its ancestors in the file + * system hierarchy. + * + * @param dir + * @throws DiskErrorException + */ public static void checkDir(File dir) throws DiskErrorException { - if (!mkdirsWithExistsCheck(dir)) - throw new DiskErrorException("can not create directory: " - + dir.toString()); - - if (!dir.isDirectory()) - throw new DiskErrorException("not a directory: " - + dir.toString()); - - if (!dir.canRead()) - throw new DiskErrorException("directory is not readable: " - + dir.toString()); - - if (!dir.canWrite()) - throw new DiskErrorException("directory is not writable: " - + dir.toString()); + checkDir(dir, null); } + /** + * Tries creating the given directory, and if needed its ancestors in the file + * system hierarchy. Depending on the permission information passed, sets + * permissions to the leaf of newly created paths. + * + * @param file + * @param fileCreationPermissionsInfo + * @throws DiskErrorException + */ + public static void checkDir(File file, + FileUtil.PermissionsInfo fileCreationPermissionsInfo) + throws DiskErrorException { + + if (!mkdirsWithExistsCheck(file, fileCreationPermissionsInfo)) { + throw new DiskErrorException("can not create directory: " + + file.toString()); + } + + if (!file.isDirectory() && !FileUtil.setPermissions(file, fileCreationPermissionsInfo)) { + throw new DiskErrorException("Cannot set permissions to directory: " + + file.toString()); + } + + if (!file.isDirectory()) + throw new DiskErrorException("not a directory: " + file.toString()); + + if (!file.canRead()) + throw new DiskErrorException("directory is not readable: " + + file.toString()); + + if (!file.canWrite()) + throw new DiskErrorException("directory is not writable: " + + file.toString()); + } }