From 9f5bc80e5cf9a5208a7ab5409e5475465759be3c Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Wed, 20 Mar 2013 16:20:49 -0700 Subject: [PATCH] HBASE-8158 backport HBASE-8140 "addDependencyJar should create jar from classes" --- .../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 82 +++++----- .../hbase/mapreduce/hadoopbackport/JarFinder.java | 170 +++++++++++++++++++++ .../mapreduce/hadoopbackport/TestJarFinder.java | 132 ++++++++++++++++ 3 files changed, 337 insertions(+), 47 deletions(-) create mode 100644 src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java create mode 100644 src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index e495259..3c2f2fb 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -24,8 +24,6 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; import java.net.URL; import java.net.URLDecoder; import java.util.ArrayList; @@ -39,11 +37,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; @@ -581,29 +580,25 @@ public class TableMapReduceUtil { * @return a jar file that contains the class, or null. * @throws IOException */ - private static String findOrCreateJar(Class my_class) + private static String findOrCreateJar(Class my_class) throws IOException { - try { - Class jarFinder = Class.forName("org.apache.hadoop.util.JarFinder"); - // hadoop-0.23 has a JarFinder class that will create the jar - // if it doesn't exist. Note that this is needed to run the mapreduce - // unit tests post-0.23, because mapreduce v2 requires the relevant jars - // to be in the mr cluster to do output, split, etc. At unit test time, - // the hbase jars do not exist, so we need to create some. Note that we - // can safely fall back to findContainingJars for pre-0.23 mapreduce. - Method m = jarFinder.getMethod("getJar", Class.class); - return (String)m.invoke(null,my_class); - } catch (InvocationTargetException ite) { - // function was properly called, but threw it's own exception - throw new IOException(ite.getCause()); - } catch (Exception e) { - // ignore all other exceptions. related to reflection failure - } + // attempt to locate an existing jar for the class. + String jar = findContainingJar(my_class); + if (null != jar && !jar.isEmpty()) { + LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar)); + return jar; + } - LOG.debug("New JarFinder: org.apache.hadoop.util.JarFinder.getJar " + - "not available. Using old findContainingJar"); - return findContainingJar(my_class); -} + // failing that, see if a jar can be build using JarFinder. Useful for some + // job configuration contexts (HBASE-8140) and also for testing on MRv2. + jar = JarFinder.getJar(my_class); + if (null != jar && !jar.isEmpty()) { + LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar)); + return jar; + } + + throw new RuntimeException("Cannot locate resource for class " + my_class.getName()); + } /** * Find a jar that contains a class of the same name, if any. @@ -616,34 +611,27 @@ public class TableMapReduceUtil { * @return a jar file that contains the class, or null. * @throws IOException */ - private static String findContainingJar(Class my_class) { + private static String findContainingJar(Class my_class) throws IOException { ClassLoader loader = my_class.getClassLoader(); String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; - try { - for(Enumeration itr = loader.getResources(class_file); - itr.hasMoreElements();) { - URL url = (URL) itr.nextElement(); - if ("jar".equals(url.getProtocol())) { - String toReturn = url.getPath(); - if (toReturn.startsWith("file:")) { - toReturn = toReturn.substring("file:".length()); - } - // URLDecoder is a misnamed class, since it actually decodes - // x-www-form-urlencoded MIME type rather than actual - // URL encoding (which the file path has). Therefore it would - // decode +s to ' 's which is incorrect (spaces are actually - // either unencoded or encoded as "%20"). Replace +s first, so - // that they are kept sacred during the decoding process. - toReturn = toReturn.replaceAll("\\+", "%2B"); - toReturn = URLDecoder.decode(toReturn, "UTF-8"); - return toReturn.replaceAll("!.*$", ""); + for (Enumeration itr = loader.getResources(class_file); itr.hasMoreElements();) { + URL url = itr.nextElement(); + if ("jar".equals(url.getProtocol())) { + String toReturn = url.getPath(); + if (toReturn.startsWith("file:")) { + toReturn = toReturn.substring("file:".length()); } + // URLDecoder is a misnamed class, since it actually decodes + // x-www-form-urlencoded MIME type rather than actual + // URL encoding (which the file path has). Therefore it would + // decode +s to ' 's which is incorrect (spaces are actually + // either unencoded or encoded as "%20"). Replace +s first, so + // that they are kept sacred during the decoding process. + toReturn = toReturn.replaceAll("\\+", "%2B"); + toReturn = URLDecoder.decode(toReturn, "UTF-8"); + return toReturn.replaceAll("!.*$", ""); } - } catch (IOException e) { - throw new RuntimeException(e); } return null; } - - } diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java new file mode 100644 index 0000000..14e7949 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java @@ -0,0 +1,170 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.hbase.mapreduce.hadoopbackport; + +import com.google.common.base.Preconditions; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLDecoder; +import java.text.MessageFormat; +import java.util.Enumeration; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Finds the Jar for a class. If the class is in a directory in the + * classpath, it creates a Jar on the fly with the contents of the directory + * and returns the path to that Jar. If a Jar is created, it is created in + * the system temporary directory. + * + * This file was forked from hadoop/common/branches/branch-2@1377176. + */ +public class JarFinder { + + private static void copyToZipStream(InputStream is, ZipEntry entry, + ZipOutputStream zos) throws IOException { + zos.putNextEntry(entry); + byte[] arr = new byte[4096]; + int read = is.read(arr); + while (read > -1) { + zos.write(arr, 0, read); + read = is.read(arr); + } + is.close(); + zos.closeEntry(); + } + + public static void jarDir(File dir, String relativePath, ZipOutputStream zos) + throws IOException { + Preconditions.checkNotNull(relativePath, "relativePath"); + Preconditions.checkNotNull(zos, "zos"); + + // by JAR spec, if there is a manifest, it must be the first entry in the + // ZIP. + File manifestFile = new File(dir, JarFile.MANIFEST_NAME); + ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME); + if (!manifestFile.exists()) { + zos.putNextEntry(manifestEntry); + new Manifest().write(new BufferedOutputStream(zos)); + zos.closeEntry(); + } else { + InputStream is = new FileInputStream(manifestFile); + copyToZipStream(is, manifestEntry, zos); + } + zos.closeEntry(); + zipDir(dir, relativePath, zos, true); + zos.close(); + } + + private static void zipDir(File dir, String relativePath, ZipOutputStream zos, + boolean start) throws IOException { + String[] dirList = dir.list(); + for (String aDirList : dirList) { + File f = new File(dir, aDirList); + if (!f.isHidden()) { + if (f.isDirectory()) { + if (!start) { + ZipEntry dirEntry = new ZipEntry(relativePath + f.getName() + "/"); + zos.putNextEntry(dirEntry); + zos.closeEntry(); + } + String filePath = f.getPath(); + File file = new File(filePath); + zipDir(file, relativePath + f.getName() + "/", zos, false); + } + else { + String path = relativePath + f.getName(); + if (!path.equals(JarFile.MANIFEST_NAME)) { + ZipEntry anEntry = new ZipEntry(path); + InputStream is = new FileInputStream(f); + copyToZipStream(is, anEntry, zos); + } + } + } + } + } + + private static void createJar(File dir, File jarFile) throws IOException { + Preconditions.checkNotNull(dir, "dir"); + Preconditions.checkNotNull(jarFile, "jarFile"); + File jarDir = jarFile.getParentFile(); + if (!jarDir.exists()) { + if (!jarDir.mkdirs()) { + throw new IOException(MessageFormat.format("could not create dir [{0}]", + jarDir)); + } + } + JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile)); + jarDir(dir, "", zos); + } + + /** + * Returns the full path to the Jar containing the class. It always return a + * JAR. + * + * @param klass class. + * + * @return path to the Jar containing the class. + */ + public static String getJar(Class klass) { + Preconditions.checkNotNull(klass, "klass"); + ClassLoader loader = klass.getClassLoader(); + if (loader != null) { + String class_file = klass.getName().replaceAll("\\.", "/") + ".class"; + try { + for (Enumeration itr = loader.getResources(class_file); + itr.hasMoreElements(); ) { + URL url = (URL) itr.nextElement(); + String path = url.getPath(); + if (path.startsWith("file:")) { + path = path.substring("file:".length()); + } + path = URLDecoder.decode(path, "UTF-8"); + if ("jar".equals(url.getProtocol())) { + path = URLDecoder.decode(path, "UTF-8"); + return path.replaceAll("!.*$", ""); + } + else if ("file".equals(url.getProtocol())) { + String klassName = klass.getName(); + klassName = klassName.replace(".", "/") + ".class"; + path = path.substring(0, path.length() - klassName.length()); + File baseDir = new File(path); + File testDir = new File(System.getProperty("test.build.dir", "target/test-dir")); + testDir = testDir.getAbsoluteFile(); + if (!testDir.exists()) { + testDir.mkdirs(); + } + File tempJar = File.createTempFile("hadoop-", "", testDir); + tempJar = new File(tempJar.getAbsolutePath() + ".jar"); + createJar(baseDir, tempJar); + return tempJar.getAbsolutePath(); + } + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + return null; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java new file mode 100644 index 0000000..fb56993 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce.hadoopbackport; + +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.Properties; +import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +/** + * This file was forked from hadoop/common/branches/branch-2@1350012. + */ +@Category(SmallTests.class) +public class TestJarFinder { + + @Test + public void testJar() throws Exception { + + //picking a class that is for sure in a JAR in the classpath + String jar = JarFinder.getJar(LogFactory.class); + Assert.assertTrue(new File(jar).exists()); + } + + private static void delete(File file) throws IOException { + if (file.getAbsolutePath().length() < 5) { + throw new IllegalArgumentException( + MessageFormat.format("Path [{0}] is too short, not deleting", + file.getAbsolutePath())); + } + if (file.exists()) { + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + delete(child); + } + } + } + if (!file.delete()) { + throw new RuntimeException( + MessageFormat.format("Could not delete path [{0}]", + file.getAbsolutePath())); + } + } + } + + @Test + public void testExpandedClasspath() throws Exception { + //picking a class that is for sure in a directory in the classpath + //in this case the JAR is created on the fly + String jar = JarFinder.getJar(TestJarFinder.class); + Assert.assertTrue(new File(jar).exists()); + } + + @Test + public void testExistingManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testExistingManifest"); + delete(dir); + dir.mkdirs(); + + File metaInfDir = new File(dir, "META-INF"); + metaInfDir.mkdirs(); + File manifestFile = new File(metaInfDir, "MANIFEST.MF"); + Manifest manifest = new Manifest(); + OutputStream os = new FileOutputStream(manifestFile); + manifest.write(os); + os.close(); + + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } + + @Test + public void testNoManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testNoManifest"); + delete(dir); + dir.mkdirs(); + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } +} -- 1.8.1