diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 2323650..19e8794 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -220,17 +220,12 @@ public boolean accept(final Path file) { final boolean resolveSymlinks; private FileContext(final AbstractFileSystem defFs, - final FsPermission theUmask, final Configuration aConf) { + final UserGroupInformation theUgi, final FsPermission theUmask, + final Configuration aConf) { defaultFS = defFs; + ugi = theUgi; umask = FsPermission.getUMask(aConf); conf = aConf; - try { - ugi = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - LOG.error("Exception in getCurrentUser: ",e); - throw new RuntimeException("Failed to get the current user " + - "while creating a FileContext", e); - } /* * Init the wd. * WorkingDir is implemented at the FileContext layer @@ -249,6 +244,16 @@ private FileContext(final AbstractFileSystem defFs, util = new Util(); // for the inner class } + private static UserGroupInformation getCurrentUser() { + try { + return UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + LOG.error("Exception in getCurrentUser: ",e); + throw new RuntimeException("Failed to get the current user " + + "while creating a FileContext", e); + } + } + /* * Remove relative part - return "absolute": * If input is relative path ("foo/bar") add wd: ie "//foo/bar" @@ -335,6 +340,18 @@ public AbstractFileSystem run() throws UnsupportedFileSystemException { * Protected Static Factory methods for getting a FileContexts * that take a AbstractFileSystem as input. To be used for testing. */ + /** + * Create a FileContext with specified FS as default using the specified + * user group information and config. + * @param defFS + * @param user + * @param aConf + * @return new FileContext with specified FS as default. + */ + protected static FileContext getFileContext(final AbstractFileSystem defFS, + final UserGroupInformation user, final Configuration aConf) { + return new FileContext(defFS, user, FsPermission.getUMask(aConf), aConf); + } /** * Create a FileContext with specified FS as default using the specified @@ -342,11 +359,11 @@ public AbstractFileSystem run() throws UnsupportedFileSystemException { * * @param defFS * @param aConf - * @return new FileContext with specifed FS as default. + * @return new FileContext with specified FS as default. */ public static FileContext getFileContext(final AbstractFileSystem defFS, final Configuration aConf) { - return new FileContext(defFS, FsPermission.getUMask(aConf), aConf); + return getFileContext(defFS, getCurrentUser(), aConf); } /** @@ -392,6 +409,18 @@ public static FileContext getFileContext() } /** + * Creates a clone of the provided FileContext. + * + * @param aFileContext the FileContext to clone + * @return a FileContext with the same FS as aFileContext + */ + public static FileContext getFileContext(final FileContext aFileContext) { + return new FileContext( + aFileContext.defaultFS, aFileContext.ugi, aFileContext.umask, + aFileContext.conf); + } + + /** * @return a FileContext for the local file system using the default config. * @throws UnsupportedFileSystemException If the file system for * {@link FsConstants#LOCAL_FS_URI} is not supported. @@ -431,7 +460,7 @@ public static FileContext getFileContext(final URI defaultFsUri, UserGroupInformation currentUser = null; AbstractFileSystem defaultAfs = null; try { - currentUser = UserGroupInformation.getCurrentUser(); + currentUser = getCurrentUser(); defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf); } catch (UnsupportedFileSystemException ex) { throw ex; @@ -439,7 +468,7 @@ public static FileContext getFileContext(final URI defaultFsUri, LOG.error(ex); throw new RuntimeException(ex); } - return getFileContext(defaultAfs, aConf); + return getFileContext(defaultAfs, currentUser, aConf); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java new file mode 100644 index 0000000..f74e27a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.StringUtils; + +public class TestFileContext { + private final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); + private FileContext fc; + + @Before + public void setUp() throws Exception { + fc = FileContext.getLocalFSFileContext(); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DIR_DEFAULT_PERM, true); + } + + @After + public void tearDown() throws Exception { + fc.delete(fileContextTestHelper.getTestRootPath(fc), true); + } + + @Test + public void testGetFileContextClone() throws IOException { + Path newWD = fileContextTestHelper.getTestRootPath(fc, "new_wd"); + + if (!fc.util().exists(newWD)) { + fc.mkdir(newWD, FileContext.DIR_DEFAULT_PERM, false); + } + + FileContext fc1 = FileContext.getFileContext(fc); + fc1.setWorkingDirectory(newWD); + fc1.setUMask(FsPermission.createImmutable((short) 0)); + + // fc and fc1 should be different object with same underlying filesystem/ugi + Assert.assertNotSame(fc, fc1); + Assert.assertSame(fc.getDefaultFileSystem(), fc1.getDefaultFileSystem()); + Assert.assertSame(fc.getUgi(), fc1.getUgi()); + Assert.assertFalse(fc1.getWorkingDirectory().equals(fc.getWorkingDirectory())); + Assert.assertFalse(fc1.getUMask().equals(fc.getUMask())); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 834b138..faae00e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -94,7 +94,7 @@ public void init() throws IOException { } @Override - public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, + public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { @@ -102,10 +102,6 @@ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, List localDirs = dirsHandler.getLocalDirs(); List logDirs = dirsHandler.getLogDirs(); - ContainerLocalizer localizer = - new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs), - RecordFactoryProvider.getRecordFactory(getConf())); - createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); @@ -118,8 +114,14 @@ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); - lfs.setWorkingDirectory(appStorageDir); - LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory()); + + FileContext localizerFileContext = FileContext.getFileContext(lfs); + localizerFileContext.setWorkingDirectory(appStorageDir); + LOG.info("Localizer CWD set to " + appStorageDir + " = " + + localizerFileContext.getWorkingDirectory()); + ContainerLocalizer localizer = + new ContainerLocalizer(localizerFileContext, user, appId, locId, + getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }