Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (revision dc735b286bb656903df49aee776d22ee0c61f860) +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (revision 6bcf7f43af749e71fcfe2b8c2db10b4dcb74c58a) @@ -34,9 +34,11 @@ import java.util.List; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import java.util.jar.JarInputStream; import java.util.jar.Manifest; import java.util.regex.Pattern; +import org.apache.commons.io.input.TeeInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileUtil; @@ -94,6 +96,71 @@ unJar(jarFile, toDir, MATCH_ANY); } + /** + * Unpack matching files from a jar. Entries inside the jar that do + * not match the given pattern will be skipped. + * + * @param jarFile the jar stream to unpack + * @param toDir the destination directory into which to unpack the jar + * @param unpackRegex the pattern to match jar entries against + * + * @throws IOException if an I/O error has occurred or toDir + * cannot be created and does not already exist + */ + public static void unJar(InputStream jarFile, File toDir, + Pattern unpackRegex) + throws IOException { + try (JarInputStream jar = new JarInputStream(jarFile)) { + int numOfFailedLastModifiedSet = 0; + do { + final JarEntry entry = jar.getNextJarEntry(); + if (entry == null) { + break; + } + if (!entry.isDirectory() && + unpackRegex.matcher(entry.getName()).matches()) { + File file = new File(toDir, entry.getName()); + ensureDirectory(file.getParentFile()); + try (OutputStream out = new FileOutputStream(file)) { + IOUtils.copyBytes(jar, out, BUFFER_SIZE); + } + if (!file.setLastModified(entry.getTime())) { + numOfFailedLastModifiedSet++; + } + } + } while (true); + if (numOfFailedLastModifiedSet > 0) { + LOG.warn("Could not set last modfied time for {} file(s)", + numOfFailedLastModifiedSet); + } + } + } + + /** + * Unpack matching files from a jar. Entries inside the jar that do + * not match the given pattern will be skipped. Keep also a copy + * of the entire jar in the same directory for backward compatibility. + * TODO remove this feature in a new release and do only unJar + * + * @param jarFile the jar stream to unpack + * @param toDir the destination directory into which to unpack the jar + * @param unpackRegex the pattern to match jar entries against + * + * @throws IOException if an I/O error has occurred or toDir + * cannot be created and does not already exist + */ + @Deprecated + public static void unJarAndSave(InputStream jarFile, File toDir, + String name, Pattern unpackRegex) + throws IOException{ + File file = new File(toDir, name); + ensureDirectory(toDir); + try (OutputStream jar = new FileOutputStream(file); + TeeInputStream teeInputStream = new TeeInputStream(jarFile, jar)) { + unJar(teeInputStream, toDir, unpackRegex); + } + } + /** * Unpack matching files from a jar. Entries inside the jar that do * not match the given pattern will be skipped. Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (revision dc735b286bb656903df49aee776d22ee0c61f860) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (revision 6bcf7f43af749e71fcfe2b8c2db10b4dcb74c58a) @@ -21,6 +21,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.concurrent.Callable; @@ -29,6 +31,7 @@ import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; @@ -54,6 +57,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.Futures; +import org.apache.hadoop.yarn.exceptions.YarnException; /** * Download a single URL to the local disk. @@ -247,9 +251,21 @@ } } - private Path copy(Path sCopy, Path dstdir) throws IOException { + /** + * Localize files. + * @param destination destination directory + * @throws IOException cannot read or write file + * @throws YarnException subcommand returned an error + */ + private void verifyAndCopy(Path destination) + throws IOException, YarnException { + final Path sCopy; + try { + sCopy = resource.getResource().toPath(); + } catch (URISyntaxException e) { + throw new IOException("Invalid resource", e); + } FileSystem sourceFs = sCopy.getFileSystem(conf); - Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + @@ -264,82 +280,107 @@ } } - FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false, - true, conf); - return dCopy; + try { + downloadAndUnpack(sCopy, destination); + } catch (RuntimeException ex) { + throw new YarnException("Parallel download failed", ex); + } + } + + /** + * Copy source path to destination with localization rules. + * @param source source path to copy. Typically HDFS + * @param destination destination path. Typically local filesystem + */ + private void downloadAndUnpack(Path source, Path destination) { + try { + FileSystem sourceFileSystem = source.getFileSystem(conf); + FileSystem destinationFileSystem = destination.getFileSystem(conf); + if (sourceFileSystem.getFileStatus(source).isDirectory()) { + FileUtil.copy( + sourceFileSystem, source, + destinationFileSystem, destination, false, + true, conf); + } else { + unpack(source, destination, sourceFileSystem, destinationFileSystem); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } - private long unpack(File localrsrc, File dst) throws IOException { - switch (resource.getType()) { - case ARCHIVE: { + /** + * Do the localization action on the input stream. + * @param source Source path + * @param destination Destination pth + * @param sourceFileSystem Source filesystem + * @param destinationFileSystem Destination filesystem + * @throws IOException Could not read or write stream + * @throws InterruptedException Operation interrupted by caller + * @throws ExecutionException Could not create thread pool execution + */ + private void unpack(Path source, Path destination, + FileSystem sourceFileSystem, + FileSystem destinationFileSystem) + throws IOException, InterruptedException, ExecutionException { + try (InputStream inputStream = sourceFileSystem.open(source)) { + File dst = new File(destination.toUri()); String lowerDst = StringUtils.toLowerCase(dst.getName()); - if (lowerDst.endsWith(".jar")) { - RunJar.unJar(localrsrc, dst); - } else if (lowerDst.endsWith(".zip")) { - FileUtil.unZip(localrsrc, dst); - } else if (lowerDst.endsWith(".tar.gz") || - lowerDst.endsWith(".tgz") || - lowerDst.endsWith(".tar")) { - FileUtil.unTar(localrsrc, dst); - } else { - LOG.warn("Cannot unpack " + localrsrc); - if (!localrsrc.renameTo(dst)) { - throw new IOException("Unable to rename file: [" + localrsrc - + "] to [" + dst + "]"); - } - } - } - break; - case PATTERN: { - String lowerDst = StringUtils.toLowerCase(dst.getName()); - if (lowerDst.endsWith(".jar")) { - String p = resource.getPattern(); - RunJar.unJar(localrsrc, dst, - p == null ? RunJar.MATCH_ANY : Pattern.compile(p)); - File newDst = new File(dst, dst.getName()); - if (!dst.exists() && !dst.mkdir()) { - throw new IOException("Unable to create directory: [" + dst + "]"); - } - if (!localrsrc.renameTo(newDst)) { - throw new IOException("Unable to rename file: [" + localrsrc - + "] to [" + newDst + "]"); - } - } else if (lowerDst.endsWith(".zip")) { - LOG.warn("Treating [" + localrsrc + "] as an archive even though it " + - "was specified as PATTERN"); - FileUtil.unZip(localrsrc, dst); - } else if (lowerDst.endsWith(".tar.gz") || - lowerDst.endsWith(".tgz") || - lowerDst.endsWith(".tar")) { - LOG.warn("Treating [" + localrsrc + "] as an archive even though it " + - "was specified as PATTERN"); - FileUtil.unTar(localrsrc, dst); - } else { - LOG.warn("Cannot unpack " + localrsrc); - if (!localrsrc.renameTo(dst)) { - throw new IOException("Unable to rename file: [" + localrsrc - + "] to [" + dst + "]"); - } - } - } - break; - case FILE: - default: - if (!localrsrc.renameTo(dst)) { - throw new IOException("Unable to rename file: [" + localrsrc - + "] to [" + dst + "]"); - } - break; - } - if(localrsrc.isFile()){ - try { - files.delete(new Path(localrsrc.toString()), false); - } catch (IOException ignore) { + switch (resource.getType()) { + case ARCHIVE: + if (lowerDst.endsWith(".jar")) { + RunJar.unJar(inputStream, dst, RunJar.MATCH_ANY); + } else if (lowerDst.endsWith(".zip")) { + FileUtil.unZip(inputStream, dst); + } else if (lowerDst.endsWith(".tar.gz") || + lowerDst.endsWith(".tgz") || + lowerDst.endsWith(".tar")) { + FileUtil.unTar(inputStream, dst, lowerDst.endsWith("gz")); + } else { + LOG.warn("Cannot unpack " + source); + try (OutputStream outputStream = + destinationFileSystem.create(destination, true)) { + IOUtils.copy(inputStream, outputStream); + } + } + break; + case PATTERN: + if (lowerDst.endsWith(".jar")) { + String p = resource.getPattern(); + if (!dst.exists() && !dst.mkdir()) { + throw new IOException("Unable to create directory: [" + dst + "]"); + } + RunJar.unJarAndSave(inputStream, dst, source.getName(), + p == null ? RunJar.MATCH_ANY : Pattern.compile(p)); + } else if (lowerDst.endsWith(".zip")) { + LOG.warn("Treating [" + source + "] as an archive even though it " + + "was specified as PATTERN"); + FileUtil.unZip(inputStream, dst); + } else if (lowerDst.endsWith(".tar.gz") || + lowerDst.endsWith(".tgz") || + lowerDst.endsWith(".tar")) { + LOG.warn("Treating [" + source + "] as an archive even though it " + + "was specified as PATTERN"); + FileUtil.unTar(inputStream, dst, lowerDst.endsWith("gz")); + } else { + LOG.warn("Cannot unpack " + source); + try (OutputStream outputStream = + destinationFileSystem.create(destination, true)) { + IOUtils.copy(inputStream, outputStream); + } + } + break; + case FILE: + default: + try (OutputStream outputStream = + destinationFileSystem.create(destination, true)) { + IOUtils.copy(inputStream, outputStream); + } + break; } - } - return 0; - // TODO Should calculate here before returning - //return FileUtil.getDU(destDir); + // TODO Should calculate here before returning + //return FileUtil.getDU(destDir); + } } @Override @@ -352,27 +393,34 @@ } if (LOG.isDebugEnabled()) { - LOG.debug("Starting to download " + sCopy); + LOG.debug(String.format("Starting to download %s %s %s", + sCopy, + resource.getType(), + resource.getPattern())); } - createDir(destDirPath, cachePerms); - final Path dst_work = new Path(destDirPath + "_tmp"); - createDir(dst_work, cachePerms); - Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName())); + final Path destinationTmp = new Path(destDirPath + "_tmp"); + createDir(destinationTmp, PRIVATE_DIR_PERMS); + Path dFinal = + files.makeQualified(new Path(destinationTmp, sCopy.getName())); try { - Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work)) - : userUgi.doAs(new PrivilegedExceptionAction() { - public Path run() throws Exception { - return files.makeQualified(copy(sCopy, dst_work)); - }; - }); - unpack(new File(dTmp.toUri()), new File(dFinal.toUri())); + if (userUgi == null) { + verifyAndCopy(dFinal); + } else { + userUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + verifyAndCopy(dFinal); + return null; + } + }); + } changePermissions(dFinal.getFileSystem(conf), dFinal); - files.rename(dst_work, destDirPath, Rename.OVERWRITE); + files.rename(destinationTmp, destDirPath, Rename.OVERWRITE); if (LOG.isDebugEnabled()) { - LOG.debug("File has been downloaded to " + - new Path(destDirPath, sCopy.getName())); + LOG.debug(String.format("File has been downloaded to %s from %s", + new Path(destDirPath, sCopy.getName()), sCopy)); } } catch (Exception e) { try { @@ -382,7 +430,7 @@ throw e; } finally { try { - files.delete(dst_work, true); + files.delete(destinationTmp, true); } catch (FileNotFoundException ignore) { } conf = null; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (revision dc735b286bb656903df49aee776d22ee0c61f860) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (revision ea38c8a9ce4a79e19e808484eb8427564a2454f2) @@ -82,6 +82,9 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +/** + * Unit test for the FSDownload class. + */ public class TestFSDownload { private static final Log LOG = LogFactory.getLog(TestFSDownload.class); @@ -90,7 +93,8 @@ private enum TEST_FILE_TYPE { TAR, JAR, ZIP, TGZ }; - + private Configuration conf = new Configuration(); + @AfterClass public static void deleteTestDir() throws IOException { FileContext fs = FileContext.getLocalFSFileContext(); @@ -132,6 +136,18 @@ FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); + ZipEntry entry = new ZipEntry("classes/1.class"); + out.putNextEntry(entry); + out.write(1); + out.write(2); + out.write(3); + out.closeEntry(); + ZipEntry entry2 = new ZipEntry("classes/2.class"); + out.putNextEntry(entry2); + out.write(1); + out.write(2); + out.write(3); + out.closeEntry(); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); @@ -256,7 +272,6 @@ @Test (timeout=10000) public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException { - Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", @@ -307,7 +322,6 @@ @Test (timeout=60000) public void testDownloadPublicWithStatCache() throws IOException, URISyntaxException, InterruptedException, ExecutionException { - final Configuration conf = new Configuration(); FileContext files = FileContext.getLocalFSFileContext(conf); Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName())); @@ -382,7 +396,6 @@ @Test (timeout=10000) public void testDownload() throws IOException, URISyntaxException, InterruptedException { - Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", @@ -438,7 +451,7 @@ FileStatus status = files.getFileStatus(localized.getParent()); FsPermission perm = status.getPermission(); assertEquals("Cache directory permissions are incorrect", - new FsPermission((short)0755), perm); + new FsPermission((short)0700), perm); status = files.getFileStatus(localized); perm = status.getPermission(); @@ -455,7 +468,6 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, URISyntaxException, InterruptedException{ - Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", @@ -530,7 +542,7 @@ } } - @Test (timeout=10000) + @Test (timeout=10000) public void testDownloadArchive() throws IOException, URISyntaxException, InterruptedException { downloadWithFileType(TEST_FILE_TYPE.TAR); @@ -542,7 +554,7 @@ downloadWithFileType(TEST_FILE_TYPE.JAR); } - @Test (timeout=10000) + @Test (timeout=10000) public void testDownloadArchiveZip() throws IOException, URISyntaxException, InterruptedException { downloadWithFileType(TEST_FILE_TYPE.ZIP); @@ -603,7 +615,6 @@ @Test (timeout=10000) public void testDirDownload() throws IOException, InterruptedException { - Configuration conf = new Configuration(); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName())); @@ -668,7 +679,6 @@ @Test (timeout=10000) public void testUniqueDestinationPath() throws Exception { - Configuration conf = new Configuration(); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName())); Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (revision c529eb4ec4f870c84fbe4273bf1b76a01706dd35) +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (revision 6bcf7f43af749e71fcfe2b8c2db10b4dcb74c58a) @@ -35,12 +35,17 @@ import java.util.Enumeration; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.jar.Attributes; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -74,6 +79,11 @@ * */ public static final int SYMLINK_NO_PRIVILEGE = 2; + /** + * Buffer size for copy the content of compressed file to new file. + */ + private static final int BUFFER_SIZE = 8_192; + /** * convert an array of FileStatus to an array of Path * @@ -576,11 +586,50 @@ } /** - * Given a File input it will unzip the file in a the unzip directory + * Given a stream input it will unzip the it in the unzip directory. + * passed as the second parameter + * @param inputStream The zip file as input + * @param toDir The unzip directory where to unzip the zip file. + * @throws IOException an exception occurred + */ + public static void unZip(InputStream inputStream, File toDir) + throws IOException { + try (ZipInputStream zip = new ZipInputStream(inputStream)) { + int numOfFailedLastModifiedSet = 0; + do { + final ZipEntry entry = zip.getNextEntry(); + if (entry == null) { + break; + } + if (!entry.isDirectory()) { + File file = new File(toDir, entry.getName()); + File parent = file.getParentFile(); + if (!parent.mkdirs() && + !parent.isDirectory()) { + throw new IOException("Mkdirs failed to create " + + parent.getAbsolutePath()); + } + try (OutputStream out = new FileOutputStream(file)) { + IOUtils.copyBytes(zip, out, BUFFER_SIZE); + } + if (!file.setLastModified(entry.getTime())) { + numOfFailedLastModifiedSet++; + } + } + } while (true); + if (numOfFailedLastModifiedSet > 0) { + LOG.warn("Could not set last modfied time for {} file(s)", + numOfFailedLastModifiedSet); + } + } + } + + /** + * Given a File input it will unzip it in the unzip directory. * passed as the second parameter * @param inFile The zip file as input * @param unzipDir The unzip directory where to unzip the zip file. - * @throws IOException + * @throws IOException An I/O exception has occurred */ public static void unZip(File inFile, File unzipDir) throws IOException { Enumeration entries; @@ -620,6 +669,103 @@ } } + /** + * Run a command and send the contents of an input stream to it. + * @param inputStream Input stream to forward to the shell command + * @param command shell command to run + * @return the exit code + * @throws IOException read or write failed + * @throws InterruptedException command interrupted + * @throws ExecutionException task submit failed + */ + private static int runCommandOnStream( + InputStream inputStream, String command) + throws IOException, InterruptedException, ExecutionException { + String shell = Shell.WINDOWS ? "cmd" : "bash"; + String cmdSwitch = Shell.WINDOWS ? "/c" : "-c"; + ExecutorService executor = null; + ProcessBuilder builder = new ProcessBuilder(); + builder.command(shell, cmdSwitch, command); + Process process = builder.start(); + Future output = null; + Future error = null; + String result = + String.format("%nEnable debug logs on %s for details", LOG.getName()); + if (LOG.isDebugEnabled()) { + executor = Executors.newFixedThreadPool(2); + output = executor.submit(() -> { + // Read until the output stream receives an EOF and closed. + try { + return + org.apache.commons.io.IOUtils.toString(process.getInputStream()); + } catch (IOException e) { + return e.getMessage(); + } finally { + process.getInputStream().close(); + } + }); + error = executor.submit(() -> { + try { + // Read until the error stream receives an EOF and closed. + return + org.apache.commons.io.IOUtils.toString(process.getErrorStream()); + } catch (IOException e) { + return e.getMessage(); + } finally { + process.getErrorStream().close(); + } + }); + } + try { + org.apache.commons.io.IOUtils.copy( + inputStream, process.getOutputStream()); + } finally { + process.getOutputStream().close(); + } + if (output != null) { + result = error.get() + "\n" + output.get(); + LOG.debug(result); + } + if (executor != null) { + executor.shutdown(); + } + return process.waitFor(); + } + + /** + * Given a Tar File as input it will untar the file in a the untar directory + * passed as the second parameter + * + * This utility will untar ".tar" files and ".tar.gz","tgz" files. + * + * @param inputStream The tar file as input. + * @param untarDir The untar directory where to untar the tar file. + * @param gzipped The input stream is gzipped + * TODO Use magic number and PusbackInputStream to identify + * @throws IOException an exception occurred + * @throws InterruptedException command interrupted + * @throws ExecutionException task submit failed + */ + public static void unTar(InputStream inputStream, File untarDir, + boolean gzipped) + throws IOException, InterruptedException, ExecutionException { + if (!untarDir.mkdirs()) { + if (!untarDir.isDirectory()) { + throw new IOException("Mkdirs failed to create " + untarDir); + } + } + + if(Shell.WINDOWS) { + // Tar is not native to Windows. Use simple Java based implementation for + // tests and simple tar archives + unTarUsingJava(inputStream, untarDir, gzipped); + } else { + // spawn tar utility to untar archive for full fledged unix behavior such + // as resolving symlinks in tar archives + unTarUsingTar(inputStream, untarDir, gzipped); + } + } + /** * Given a Tar File as input it will untar the file in a the untar directory * passed as the second parameter @@ -650,6 +796,28 @@ } } + private static void unTarUsingTar(InputStream inFile, File untarDir, + boolean gzipped) + throws IOException, InterruptedException, ExecutionException { + StringBuilder untarCommand = new StringBuilder(); + if (gzipped) { + untarCommand.append("gzip -dc | ("); + } + untarCommand.append("cd '"); + untarCommand.append(FileUtil.makeShellPath(untarDir)); + untarCommand.append("' ; "); + untarCommand.append("tar -x "); + + if (gzipped) { + untarCommand.append(")"); + } + int exitcode = runCommandOnStream(inFile, untarCommand.toString()); + if (exitcode != 0) { + throw new IOException("Error untarring file " + inFile + + ". Tar process exited with exit code " + exitcode); + } + } + private static void unTarUsingTar(File inFile, File untarDir, boolean gzipped) throws IOException { StringBuffer untarCommand = new StringBuffer(); @@ -690,6 +858,29 @@ inputStream = new BufferedInputStream(new FileInputStream(inFile)); } + tis = new TarArchiveInputStream(inputStream); + + for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { + unpackEntries(tis, entry, untarDir); + entry = tis.getNextTarEntry(); + } + } finally { + IOUtils.cleanupWithLogger(LOG, tis, inputStream); + } + } + + private static void unTarUsingJava(InputStream inputStream, File untarDir, + boolean gzipped) throws IOException { + TarArchiveInputStream tis = null; + try { + if (gzipped) { + inputStream = new BufferedInputStream(new GZIPInputStream( + inputStream)); + } else { + inputStream = + new BufferedInputStream(inputStream); + } + tis = new TarArchiveInputStream(inputStream); for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {