Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (date 1517549636000) +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (date 1517550910000) @@ -38,6 +38,7 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; +import com.google.common.io.NullOutputStream; import org.apache.commons.io.input.TeeInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -131,6 +132,9 @@ LOG.warn("Could not set last modfied time for {} file(s)", numOfFailedLastModifiedSet); } + // ZipInputStream does not need the end of the file. Let's read it out. + // This helps with an additional TeeInputStream on the input. + IOUtils.copyBytes(inputStream, new NullOutputStream(), 8_192); } } Index: hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java (date 1517549636000) +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java (date 1517550910000) @@ -24,8 +24,12 @@ import static org.mockito.Mockito.when; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.util.Random; +import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.regex.Pattern; import java.util.zip.ZipEntry; @@ -33,6 +37,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -114,6 +119,59 @@ new File(unjarDir, FOOBAZ_TXT).exists()); } + private File generateBigJar(File dir) throws Exception { + File file = new File(dir, "job.jar"); + try(JarOutputStream stream = new JarOutputStream( + new FileOutputStream(file))) { + Random r = new Random(100); + for (int i = 0; i < 10; ++i) { + JarEntry entry = new JarEntry("f" + Integer.toString(i)); + stream.putNextEntry(entry); + for (int j=0; j < 756; ++j) { + stream.write(r.nextInt() & 0xFF); + } + stream.closeEntry(); + } + stream.close(); + } + return file; + } + + /** + * Test unjarring a big file. This checks appending the remainder of the file + * to the tee output stream in RunJar.unJarAndSave. + */ + @Test + public void testBigJar() throws Exception { + File dir = new File(TEST_ROOT_DIR, "testUnJarWithPattern2"); + Assert.assertTrue(dir.mkdirs()); + File input = generateBigJar(dir); + File output = new File(dir, "job2.jar"); + try { + try (InputStream is = new FileInputStream(input)) { + RunJar.unJarAndSave(is, dir, "job2.jar", Pattern.compile(".*")); + } + Assert.assertEquals(input.length(), output.length()); + File[] list = dir.listFiles(); + if (list != null) { + for (File f : list) { + if (f.getName().startsWith("f")) { + Assert.assertEquals(756, f.length()); + } + } + } + } finally { + // Clean up + File[] list = dir.listFiles(); + if (list != null) { + for (File f : list) { + f.delete(); + } + } + dir.delete(); + } + } + @Test public void testUnJarDoesNotLooseLastModify() throws Exception { File unjarDir = getUnjarDir("unjar-lastmod");