diff --git src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index 2fdaf6f..6e8f48d 100644 --- src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -63,6 +63,10 @@ public abstract class CoprocessorHost { public static final String WAL_COPROCESSOR_CONF_KEY = "hbase.coprocessor.wal.classes"; + //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/ + private static final String COPROCESSOR_JARS_DIR = File.separator + + "coprocessor" + File.separator + "jars" + File.separator; + private static final Log LOG = LogFactory.getLog(CoprocessorHost.class); /** Ordered set of loaded coprocessors with lock */ protected SortedSet coprocessors = @@ -112,7 +116,7 @@ public abstract class CoprocessorHost { protected void loadSystemCoprocessors(Configuration conf, String confKey) { Class implClass = null; - // load default coprocessors from configure file + // load default coprocessors from configure file String[] defaultCPClasses = conf.getStrings(confKey); if (defaultCPClasses == null || defaultCPClasses.length == 0) return; @@ -156,7 +160,7 @@ public abstract class CoprocessorHost { public E load(Path path, String className, int priority, Configuration conf) throws IOException { Class implClass = null; - LOG.debug("Loading coprocessor class " + className + " with path " + + LOG.debug("Loading coprocessor class " + className + " with path " + path + " and priority " + priority); // Have we already loaded the class, perhaps from an earlier region open @@ -178,13 +182,13 @@ public abstract class CoprocessorHost { if (!path.toString().endsWith(".jar")) { throw new IOException(path.toString() + ": not a jar file?"); } - FileSystem fs = path.getFileSystem(HBaseConfiguration.create()); - Path dst = new Path(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + FileSystem fs = path.getFileSystem(this.conf); + File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR); + parentDir.mkdirs(); + File dst = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + ".jar"); - fs.copyToLocalFile(path, dst); - File tmpLocal = new File(dst.toString()); - tmpLocal.deleteOnExit(); + fs.copyToLocalFile(path, new Path(dst.toString())); + dst.deleteOnExit(); // TODO: code weaving goes here @@ -198,15 +202,14 @@ public abstract class CoprocessorHost { // unsurprisingly wants URLs, not URIs; so we will use the deprecated // method which returns URLs for as long as it is available List paths = new ArrayList(); - paths.add(new File(dst.toString()).getCanonicalFile().toURL()); + paths.add(dst.getCanonicalFile().toURL()); - JarFile jarFile = new JarFile(dst.toString()); + JarFile jarFile = new JarFile(dst); Enumeration entries = jarFile.entries(); while (entries.hasMoreElements()) { JarEntry entry = entries.nextElement(); if (entry.getName().matches("/lib/[^/]+\\.jar")) { - File file = new File(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + File file = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5)); IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true); file.deleteOnExit(); diff --git src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 4beafb2..68ba1c5 100644 --- src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -61,6 +61,7 @@ public class MasterCoprocessorHost private MasterServices masterServices; MasterCoprocessorHost(final MasterServices services, final Configuration conf) { + this.conf = conf; this.masterServices = services; loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY); } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 0f61539..e0df6a1 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -136,6 +135,7 @@ public class RegionCoprocessorHost */ public RegionCoprocessorHost(final HRegion region, final RegionServerServices rsServices, final Configuration conf) { + this.conf = conf; this.rsServices = rsServices; this.region = region; this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode()); @@ -424,7 +424,7 @@ public class RegionCoprocessorHost * Called prior to rewriting the store files selected for compaction * @param store the store being compacted * @param scanner the scanner used to read store data during compaction - * @throws IOException + * @throws IOException */ public InternalScanner preCompact(Store store, InternalScanner scanner) throws IOException { ObserverContext ctx = null; @@ -497,7 +497,7 @@ public class RegionCoprocessorHost /** * Invoked before a memstore flush - * @throws IOException + * @throws IOException */ public void preFlush() throws IOException { ObserverContext ctx = null; @@ -606,7 +606,7 @@ public class RegionCoprocessorHost * Invoked just after a split * @param l the new left-hand daughter region * @param r the new right-hand daughter region - * @throws IOException + * @throws IOException */ public void postSplit(HRegion l, HRegion r) throws IOException { ObserverContext ctx = null; diff --git src/main/resources/hbase-default.xml src/main/resources/hbase-default.xml index 44ee689..94da414 100644 --- src/main/resources/hbase-default.xml +++ src/main/resources/hbase-default.xml @@ -49,7 +49,7 @@ hbase.tmp.dir - /tmp/hbase-${user.name} + ${java.io.tmpdir}/hbase-${user.name} Temporary directory on the local filesystem. Change this setting to point to a location more permanent than '/tmp' (The '/tmp' directory is often cleared on @@ -57,6 +57,13 @@ + hbase.local.dir + ${hbase.tmp.dir}/local/ + Directory on the local filesystem to be used + as a local storage. + + + hbase.master.info.port 60010 The port for the HBase Master web UI. diff --git src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index f5cc627..d254ef8 100644 --- src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -347,6 +347,10 @@ public class HBaseTestingUtility { createSubDirAndSystemProperty( "mapred.working.dir", testPath, "mapred-working-dir"); + + createSubDir( + "hbase.local.dir", + testPath, "hbase-local-dir"); } private void createSubDir(String propertyName, Path parent, String subDirName){ diff --git src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index bea1fe1..75f4e80 100644 --- src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -173,7 +173,7 @@ public class TestClassLoading { // the classpath is {hbaseSrc}/target/classes. String currentDir = new File(".").getAbsolutePath(); String classpath = - currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" + + currentDir + File.separator + "target"+ File.separator + "classes" + System.getProperty("path.separator") + System.getProperty("surefire.test.class.path"); options.add(classpath); @@ -265,6 +265,10 @@ public class TestClassLoading { assertTrue("Configuration key 'k3' was missing on a region", found2_k3); } + private String getLocalPath(File file) { + return new Path(file.toURI()).toString(); + } + @Test // HBASE-3516: Test CP Class loading from local file system public void testClassLoadingFromLocalFS() throws Exception { @@ -273,7 +277,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName3); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -299,7 +303,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName4); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -337,9 +341,9 @@ public class TestClassLoading { String cpKey2 = " Coprocessor$2 "; String cpKey3 = " coprocessor$03 "; - String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" + + String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" + Coprocessor.PRIORITY_USER; - String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | "; + String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | "; // load from default class loader String cpValue3 = " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v "; @@ -354,13 +358,13 @@ public class TestClassLoading { htd.setValue(cpKey3, cpValue3); // add 2 coprocessor by using new htd.addCoprocessor() api - htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()), + htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)), Coprocessor.PRIORITY_USER, null); Map kvs = new HashMap(); kvs.put("k1", "v1"); kvs.put("k2", "v2"); kvs.put("k3", "v3"); - htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()), + htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)), Coprocessor.PRIORITY_USER, kvs); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); @@ -433,7 +437,7 @@ public class TestClassLoading { JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName()); jarAdd.setTime(jarFile.lastModified()); out.putNextEntry(jarAdd); - + // Write file to archive FileInputStream in = new FileInputStream(jarFile); while (true) {