diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 37c81eca8996b1327c711894f92910fb150e9073..ebd1d300c3c2238ebcf540f7f0aa8ee22b9de8c3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -750,6 +750,13 @@ private static void addDeprecatedKeys() { public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; + /** Default permissions for container logs. */ + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX = + NM_PREFIX + "default-container-executor."; + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS = + NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions"; + public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT = "710"; + public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = NM_PREFIX + "resourcemanager.minimum.version"; public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 49cced6d807d17cd71690d690e72f33ecf352bdc..a4b3c4d0985da4623d860ff87c849d2dc654d4a6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1067,6 +1067,16 @@ + + The permissions settings used for the creation of container + directories when using DefaultContainerExecutor. This follows + standard user/group/all permissions format. + + yarn.nodemanager.default-container-executor.log-dirs.permissions + 710 + + + Whether to enable log aggregation. Log aggregation collects each container's logs and moves these logs onto a file-system, for e.g. HDFS, after the application completes. Users can configure the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 9ccde5f45ca614c84c605769ba62624c83576e16..49398e47b426ee78a91fcdcbb611599f79ad4188 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -74,6 +74,8 @@ protected final FileContext lfs; + private String logDirPermissions = null; + public DefaultContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); @@ -509,9 +511,6 @@ public void deleteAsUser(DeletionAsUserContext ctx) /** Permissions for user app dir. * $local.dir/usercache/$user/appcache/$appId */ static final short APPDIR_PERM = (short)0710; - /** Permissions for user log dir. - * $logdir/$user/$appId */ - static final short LOGDIR_PERM = (short)0710; private long getDiskFreeSpace(Path base) throws IOException { return lfs.getFsStatus(base).getRemaining(); @@ -702,7 +701,8 @@ void createAppLogDirs(String appId, List logDirs, String user) throws IOException { boolean appLogDirStatus = false; - FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission appLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid Path appLogDir = new Path(rootLogDir, appId); @@ -727,7 +727,8 @@ void createContainerLogDirs(String appId, String containerId, List logDirs, String user) throws IOException { boolean containerLogDirStatus = false; - FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM); + FsPermission containerLogDirPerms = new + FsPermission(getLogDirPermissions()); for (String rootLogDir : logDirs) { // create $log.dir/$appid/$containerid Path appLogDir = new Path(rootLogDir, appId); @@ -750,6 +751,27 @@ void createContainerLogDirs(String appId, String containerId, } /** + * Return default container log directory permissions. + */ + @VisibleForTesting + public String getLogDirPermissions() { + if (this.logDirPermissions==null) { + this.logDirPermissions = getConf().get( + YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, + YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT); + } + return this.logDirPermissions; + } + + /** + * Clear the internal variable for repeatable testing. + */ + @VisibleForTesting + public void clearLogDirPermissions() { + this.logDirPermissions = null; + } + + /** * @return the list of paths of given local directories */ private static List getPaths(List dirs) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index 4404a7c0106b9cf2d1a21724a2da89299a61e8d5..40e6c1bbd3ede1060fdbbfbb45c534b5c4128256 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -168,8 +168,7 @@ public void testDirPermissions() throws Exception { DefaultContainerExecutor.FILECACHE_PERM); final FsPermission appDirPerm = new FsPermission( DefaultContainerExecutor.APPDIR_PERM); - final FsPermission logDirPerm = new FsPermission( - DefaultContainerExecutor.LOGDIR_PERM); + List localDirs = new ArrayList(); localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString()); localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString()); @@ -181,6 +180,7 @@ public void testDirPermissions() throws Exception { conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext lfs = FileContext.getLocalFSFileContext(conf); DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); + executor.setConf(conf); executor.init(); try { @@ -208,11 +208,20 @@ public void testDirPermissions() throws Exception { Assert.assertEquals(appDirPerm, stats.getPermission()); } - executor.createAppLogDirs(appId, logDirs, user); + String[] permissionsArray = { "000", "111", "555", "710", "777" }; + + for (int i = 0; i < permissionsArray.length; ++i) { + conf.set(YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, permissionsArray[i]); + executor.clearLogDirPermissions(); + FsPermission logDirPerm = new FsPermission( + executor.getLogDirPermissions()); + executor.createAppLogDirs(appId, logDirs, user); - for (String dir : logDirs) { - FileStatus stats = lfs.getFileStatus(new Path(dir, appId)); - Assert.assertEquals(logDirPerm, stats.getPermission()); + for (String dir : logDirs) { + FileStatus stats = lfs.getFileStatus(new Path(dir, appId)); + Assert.assertEquals(logDirPerm, stats.getPermission()); + lfs.delete(new Path(dir, appId), true); + } } } finally { deleteTmpFiles();