diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 37c81eca8996b1327c711894f92910fb150e9073..8059f3ec0a77950b7550d455aea20b1cd5b4d29d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -750,6 +750,11 @@ private static void addDeprecatedKeys() { public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; + /** Default permissions for container logs. */ + public static final String NM_LOG_DIRS_PERMISSIONS = + NM_LOG_DIRS + ".permissions"; + public static final String NM_LOG_DIRS_PERMISSIONS_DEFAULT = "710"; + public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = NM_PREFIX + "resourcemanager.minimum.version"; public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 49cced6d807d17cd71690d690e72f33ecf352bdc..0cc31ae2ba0a5d53ce01ad851a523cdadb060b15 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1067,6 +1067,15 @@ + + The permissions settings used for the creation container directories. + This follows standard user/group/all permissions format. + + yarn.nodemanager.log-dirs.permissions + 710 + + + Whether to enable log aggregation. Log aggregation collects each container's logs and moves these logs onto a file-system, for e.g. HDFS, after the application completes. Users can configure the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 9ccde5f45ca614c84c605769ba62624c83576e16..de34f15d05d1abed776d355e5bfc2d6a341d5861 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -74,6 +74,8 @@ protected final FileContext lfs; + private String logDirPermissions = null; + public DefaultContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); @@ -509,9 +511,6 @@ public void deleteAsUser(DeletionAsUserContext ctx) /** Permissions for user app dir. * $local.dir/usercache/$user/appcache/$appId */ static final short APPDIR_PERM = (short)0710; - /** Permissions for user log dir. - * $logdir/$user/$appId */ - static final short LOGDIR_PERM = (short)0710; private long getDiskFreeSpace(Path base) throws IOException { return lfs.getFsStatus(base).getRemaining(); @@ -702,7 +701,8 @@ void createAppLogDirs(String appId, List logDirs, String user) throws IOException { boolean appLogDirStatus = false; - FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission appLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid Path appLogDir = new Path(rootLogDir, appId); @@ -727,7 +727,8 @@ void createContainerLogDirs(String appId, String containerId, List logDirs, String user) throws IOException { boolean containerLogDirStatus = false; - FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission containerLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid/$containerid Path appLogDir = new Path(rootLogDir, appId); @@ -750,6 +751,19 @@ void createContainerLogDirs(String appId, String containerId, } /** + * Return default container log directory permissions. + */ + @VisibleForTesting + public String getLogDirPermissions() { + if (this.logDirPermssions==null) { + this.logDirPermssions = getConf() + .get(YarnConfiguration.NM_LOG_DIRS_PERMISSIONS, + YarnConfiguration.NM_LOG_DIRS_PERMISSIONS_DEFAULT); + } + return this.logDirPermssions; + } + + /** * @return the list of paths of given local directories */ private static List getPaths(List dirs) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index 4404a7c0106b9cf2d1a21724a2da89299a61e8d5..eaea579727b505a7bd5d5b0fd6d489fc983f702b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -168,8 +168,7 @@ public void testDirPermissions() throws Exception { DefaultContainerExecutor.FILECACHE_PERM); final FsPermission appDirPerm = new FsPermission( DefaultContainerExecutor.APPDIR_PERM); - final FsPermission logDirPerm = new FsPermission( - DefaultContainerExecutor.LOGDIR_PERM); + List localDirs = new ArrayList(); localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString()); localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString()); @@ -179,10 +178,15 @@ public void testDirPermissions() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + conf.set(YarnConfiguration.NM_LOG_DIRS_PERM, "710"); FileContext lfs = FileContext.getLocalFSFileContext(conf); DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); + executor.setConf(conf); executor.init(); + final FsPermission logDirPerm = new FsPermission( + executor.getLogDirPermissions()); + try { executor.createUserLocalDirs(localDirs, user); executor.createUserCacheDirs(localDirs, user);