diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index bbda2e4..92c4c22 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -1261,4 +1261,5 @@ public void testCompareFsDirectories() throws Exception { assertEquals(FileUtil.compareFs(fs3,fs4),true); assertEquals(FileUtil.compareFs(fs1,fs6),false); } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 6e59574..471f88a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -275,12 +275,15 @@ private long unpack(File localrsrc, File dst) throws IOException { String lowerDst = StringUtils.toLowerCase(dst.getName()); if (lowerDst.endsWith(".jar")) { RunJar.unJar(localrsrc, dst); + moveOriginalFileToDir(localrsrc, new File(dst, dst.getName())); } else if (lowerDst.endsWith(".zip")) { FileUtil.unZip(localrsrc, dst); + moveOriginalFileToDir(localrsrc, new File(dst, dst.getName())); } else if (lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz") || lowerDst.endsWith(".tar")) { FileUtil.unTar(localrsrc, dst); + moveOriginalFileToDir(localrsrc, new File(dst, dst.getName())); } else { LOG.warn("Cannot unpack " + localrsrc); if (!localrsrc.renameTo(dst)) { @@ -308,12 +311,14 @@ private long unpack(File localrsrc, File dst) throws IOException { LOG.warn("Treating [" + localrsrc + "] as an archive even though it " + "was specified as PATTERN"); FileUtil.unZip(localrsrc, dst); + moveOriginalFileToDir(localrsrc, new File(dst, dst.getName())); } else if (lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz") || lowerDst.endsWith(".tar")) { LOG.warn("Treating [" + localrsrc + "] as an archive even though it " + "was specified as PATTERN"); FileUtil.unTar(localrsrc, dst); + moveOriginalFileToDir(localrsrc, new File(dst, dst.getName())); } else { LOG.warn("Cannot unpack " + localrsrc); if (!localrsrc.renameTo(dst)) { @@ -342,6 +347,13 @@ private long unpack(File localrsrc, File dst) throws IOException { //return FileUtil.getDU(destDir); } + private void moveOriginalFileToDir(File srcFile, File dstFile) { + // The dest dir might not exist while there's no file to unpack + File parent = new File(dstFile.getParent()); + parent.mkdirs(); + srcFile.renameTo(dstFile); + } + @Override public Path call() throws Exception { final Path sCopy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java index 877dd08..8837082 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assume.assumeTrue; import java.io.File; @@ -52,6 +53,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.api.records.URL; import org.junit.Assert; @@ -476,7 +478,6 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, int size = rand.nextInt(512) + 512; LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; Path p = new Path(basedir, "" + 1); - String strFileName = ""; LocalResource rsrc = null; switch (fileType) { case TAR: @@ -488,7 +489,6 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, break; case ZIP: rsrc = createZipFile(files, p, size, rand, vis); - strFileName = p.getName() + ".ZIP"; break; case TGZ: rsrc = createTgzFile(files, p, size, rand, vis); @@ -511,17 +511,7 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( filestatus.getPath()); for (FileStatus childfile : childFiles) { - if(strFileName.endsWith(".ZIP") && - childfile.getPath().getName().equals(strFileName) && - !childfile.isDirectory()) { - Assert.fail("Failure...After unzip, there should have been a" + - " directory formed with zip file name but found a file. " - + childfile.getPath()); - } - if (childfile.getPath().getName().startsWith("tmp")) { - Assert.fail("Tmp File should not have been there " - + childfile.getPath()); - } + checkFile(rsrc, childfile); } } } @@ -530,7 +520,80 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, } } - @Test (timeout=10000) + private void checkFile(LocalResource resource, FileStatus fileStatus) + throws URISyntaxException, IOException { + // Only check file which equals resource name for the moment + if (!fileStatus.getPath().getName().equals( + resource.getResource().toPath().getName())) { + return; + } + if (fileStatus.getPath().getName().startsWith("tmp")) { + Assert.fail("Tmp File should not have been there " + + fileStatus.getPath()); + } + if (isUnpackedType(resource)) { + // e.g. Archives .zip, .tar.gz, tgz + checkUnpackedType(resource, fileStatus); + } else { + checkNormalResource(resource, fileStatus); + } + } + + private void checkNormalResource( + LocalResource resource, FileStatus fileStatus) { + assertFalse(fileStatus.getPath() + " should not be dir!", + fileStatus.isDirectory()); + } + + private void checkUnpackedType( + LocalResource resource, FileStatus fileStatus) throws IOException { + if (!fileStatus.isDirectory()) { + Assert.fail("Failure...After unzip, there should have been a" + + " directory formed with zip file name but found a file. " + + fileStatus.getPath()); + return; + } + // E.g. test.zip with file1 inside. The final dir should be like + // test.zip/{test.zip, file1}. We need this file to upload + Path originalFilePath = new Path( + fileStatus.getPath(), fileStatus.getPath().getName()); + assertTrue( + "Original unpacked file" + originalFilePath + +" dosen't exist", + originalFilePath.getFileSystem(new Configuration()). + exists(originalFilePath)); + } + + protected boolean isUnpackedType(LocalResource resource) + throws URISyntaxException { + String lowerNameStr = StringUtils.toLowerCase(resource.getResource(). + toPath().getName()); + switch (resource.getType()) { + case ARCHIVE: + if (lowerNameStr.endsWith(".jar") + || lowerNameStr.endsWith(".zip") + || lowerNameStr.endsWith(".tar.gz") + || lowerNameStr.endsWith(".tgz") + || lowerNameStr.endsWith(".tar")) { + return true; + } else { + return false; + } + case PATTERN: + if (lowerNameStr.endsWith(".jar") + || lowerNameStr.endsWith(".zip") + || lowerNameStr.endsWith(".tar.gz") + || lowerNameStr.endsWith(".tgz") + || lowerNameStr.endsWith(".tar")) { + return true; + } else { + return false; + } default: + return false; + } + } + + @Test (timeout=10000) public void testDownloadArchive() throws IOException, URISyntaxException, InterruptedException { downloadWithFileType(TEST_FILE_TYPE.TAR);