diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index 49d4c0f..29be181 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -69,6 +69,10 @@ public abstract class CoprocessorHost { public static final String WAL_COPROCESSOR_CONF_KEY = "hbase.coprocessor.wal.classes"; + //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/ + private static final String COPROCESSOR_JARS_DIR = File.separator + + "coprocessor" + File.separator + "jars" + File.separator; + private static final Log LOG = LogFactory.getLog(CoprocessorHost.class); /** Ordered set of loaded coprocessors with lock */ protected SortedSet coprocessors = @@ -118,7 +122,7 @@ public abstract class CoprocessorHost { protected void loadSystemCoprocessors(Configuration conf, String confKey) { Class implClass = null; - // load default coprocessors from configure file + // load default coprocessors from configure file String[] defaultCPClasses = conf.getStrings(confKey); if (defaultCPClasses == null || defaultCPClasses.length == 0) return; @@ -162,7 +166,7 @@ public abstract class CoprocessorHost { public E load(Path path, String className, int priority, Configuration conf) throws IOException { Class implClass = null; - LOG.debug("Loading coprocessor class " + className + " with path " + + LOG.debug("Loading coprocessor class " + className + " with path " + path + " and priority " + priority); // Have we already loaded the class, perhaps from an earlier region open @@ -184,13 +188,13 @@ public abstract class CoprocessorHost { if (!path.toString().endsWith(".jar")) { throw new IOException(path.toString() + ": not a jar file?"); } - FileSystem fs = path.getFileSystem(HBaseConfiguration.create()); - Path dst = new Path(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + FileSystem fs = path.getFileSystem(this.conf); + File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR); + parentDir.mkdirs(); + File dst = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + ".jar"); - fs.copyToLocalFile(path, dst); - File tmpLocal = new File(dst.toString()); - tmpLocal.deleteOnExit(); + fs.copyToLocalFile(path, new Path(dst.toString())); + dst.deleteOnExit(); // TODO: code weaving goes here @@ -204,15 +208,14 @@ public abstract class CoprocessorHost { // unsurprisingly wants URLs, not URIs; so we will use the deprecated // method which returns URLs for as long as it is available List paths = new ArrayList(); - paths.add(new File(dst.toString()).getCanonicalFile().toURL()); + paths.add(dst.getCanonicalFile().toURL()); - JarFile jarFile = new JarFile(dst.toString()); + JarFile jarFile = new JarFile(dst); Enumeration entries = jarFile.entries(); while (entries.hasMoreElements()) { JarEntry entry = entries.nextElement(); if (entry.getName().matches("/lib/[^/]+\\.jar")) { - File file = new File(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + File file = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5)); IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true); file.deleteOnExit(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 2e228a2..4d4fa8c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -62,6 +62,7 @@ public class MasterCoprocessorHost private MasterServices masterServices; MasterCoprocessorHost(final MasterServices services, final Configuration conf) { + this.conf = conf; this.masterServices = services; loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index ed14364..81d1794 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -36,7 +36,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -136,6 +135,7 @@ public class RegionCoprocessorHost */ public RegionCoprocessorHost(final HRegion region, final RegionServerServices rsServices, final Configuration conf) { + this.conf = conf; this.rsServices = rsServices; this.region = region; this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode()); @@ -428,7 +428,7 @@ public class RegionCoprocessorHost * Called prior to rewriting the store files selected for compaction * @param store the store being compacted * @param scanner the scanner used to read store data during compaction - * @throws IOException + * @throws IOException */ public InternalScanner preCompact(HStore store, InternalScanner scanner) throws IOException { ObserverContext ctx = null; @@ -501,7 +501,7 @@ public class RegionCoprocessorHost /** * Invoked before a memstore flush - * @throws IOException + * @throws IOException */ public void preFlush() throws IOException { ObserverContext ctx = null; @@ -631,7 +631,7 @@ public class RegionCoprocessorHost * Invoked just after a split * @param l the new left-hand daughter region * @param r the new right-hand daughter region - * @throws IOException + * @throws IOException */ public void postSplit(HRegion l, HRegion r) throws IOException { ObserverContext ctx = null; diff --git hbase-server/src/main/resources/hbase-default.xml hbase-server/src/main/resources/hbase-default.xml index a986cee..dc8f3e9 100644 --- hbase-server/src/main/resources/hbase-default.xml +++ hbase-server/src/main/resources/hbase-default.xml @@ -49,7 +49,7 @@ hbase.tmp.dir - /tmp/hbase-${user.name} + ${java.io.tmpdir}/hbase-${user.name} Temporary directory on the local filesystem. Change this setting to point to a location more permanent than '/tmp' (The '/tmp' directory is often cleared on @@ -57,6 +57,13 @@ + hbase.local.dir + ${hbase.tmp.dir}/local/ + Directory on the local filesystem to be used + as a local storage. + + + hbase.master.info.port 60010 The port for the HBase Master web UI. diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index a006a9b..786105a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -351,6 +351,10 @@ public class HBaseTestingUtility { createSubDirAndSystemProperty( "mapred.working.dir", testPath, "mapred-working-dir"); + + createSubDir( + "hbase.local.dir", + testPath, "hbase-local-dir"); } private void createSubDir(String propertyName, Path parent, String subDirName){ diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index ea3afc0..b292076 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -174,7 +174,7 @@ public class TestClassLoading { // the classpath is {hbaseSrc}/target/classes. String currentDir = new File(".").getAbsolutePath(); String classpath = - currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" + + currentDir + File.separator + "target"+ File.separator + "classes" + System.getProperty("path.separator") + // Note that the below trick only works if mvn is running the test; // doesn't work in eclipse for example. @@ -270,6 +270,10 @@ public class TestClassLoading { assertTrue("Configuration key 'k3' was missing on a region", found2_k3); } + private String getLocalPath(File file) { + return new Path(file.toURI()).toString(); + } + @Test // HBASE-3516: Test CP Class loading from local file system public void testClassLoadingFromLocalFS() throws Exception { @@ -278,7 +282,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName3); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -304,7 +308,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName4); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -342,9 +346,9 @@ public class TestClassLoading { String cpKey2 = " Coprocessor$2 "; String cpKey3 = " coprocessor$03 "; - String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" + + String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" + Coprocessor.PRIORITY_USER; - String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | "; + String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | "; // load from default class loader String cpValue3 = " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v "; @@ -359,13 +363,13 @@ public class TestClassLoading { htd.setValue(cpKey3, cpValue3); // add 2 coprocessor by using new htd.addCoprocessor() api - htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()), + htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)), Coprocessor.PRIORITY_USER, null); Map kvs = new HashMap(); kvs.put("k1", "v1"); kvs.put("k2", "v2"); kvs.put("k3", "v3"); - htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()), + htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)), Coprocessor.PRIORITY_USER, kvs); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); @@ -440,7 +444,7 @@ public class TestClassLoading { JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName()); jarAdd.setTime(jarFile.lastModified()); out.putNextEntry(jarAdd); - + // Write file to archive FileInputStream in = new FileInputStream(jarFile); while (true) {