commit b823f11bcfa6a36174ea1855d9b303e66e6e58b2 Author: Gera Shegalov Date: Thu Dec 26 17:06:19 2013 -0800 YARN-1542.v03.patch diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java index 86909e7..d6290cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java @@ -50,21 +50,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalDirAllocator; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.viewfs.ConfigUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.AfterClass; import org.junit.Test; @@ -583,4 +577,62 @@ public void testUniqueDestinationPath() throws Exception { // destination directory (passed as an argument) + file name. Assert.assertEquals(destPath, rPath.get().getParent()); } -} \ No newline at end of file + + private void testPublicResourceInternal(Configuration conf, + FileContext srcFileCtx, FileContext localFileCtx, Path basedir, + Path cacheDir) throws Exception { + + localFileCtx.mkdir(basedir, null, true); + srcFileCtx.setWorkingDirectory(cacheDir); + + // create a public cache file + // + final Random rand = new Random(); + final long seed = rand.nextLong(); + rand.setSeed(seed); + System.out.println("SEED: " + seed); + final Path cacheFile = srcFileCtx.makeQualified( + new Path(rand.nextLong() + "cache.txt")); + final LocalResource rsrc = createFile(localFileCtx, cacheFile, 100, rand, + LocalResourceVisibility.PUBLIC); + + // set destination path + // + final Path destDir = new Path(basedir, "dest"); + localFileCtx.mkdir(destDir, null, false); + conf.setStrings(getClass().getName(), destDir.toString()); + final LocalDirAllocator dirs = new LocalDirAllocator(getClass().getName()); + final Path destPath = dirs.getLocalPathForWrite(destDir.toString(), conf); + final FSDownload fsd = new FSDownload(localFileCtx, + UserGroupInformation.getCurrentUser(), conf, destPath, rsrc); + fsd.call(); + } + + @Test + public void testPublicResourceViewFs() throws Exception { + final Configuration conf = new Configuration(); + final FileContext localFileCtx = FileContext.getLocalFSFileContext(conf); + final Path basedir = localFileCtx.makeQualified( + new Path("target", getClass().getSimpleName() + + "-testPublicResourceViewFs")); + final String cachePathStr = "/cacheRoot"; + ConfigUtil.addLink(conf, cachePathStr, basedir.toUri()); + final FileContext viewFileCtx = FileContext.getFileContext( + FsConstants.VIEWFS_URI, conf); + testPublicResourceInternal(conf, viewFileCtx, localFileCtx, basedir, + new Path(cachePathStr)); + } + + @Test + public void testPublicResourceLocal() throws Exception { + final Configuration conf = new Configuration(); + final FileContext localFileCtx = FileContext.getLocalFSFileContext(conf); + final Path basedir = localFileCtx.makeQualified( + new Path("target", getClass().getSimpleName() + + "-testPublicResourceLocal")); + final Path cacheRoot = new Path(basedir, "cacheRoot"); + localFileCtx.mkdir(cacheRoot, null, true); + testPublicResourceInternal(conf, localFileCtx, localFileCtx, basedir, + cacheRoot); + } +}