commit 5e408aedac4d22226b6e20d6f6f41ef91dce2caf Author: stack Date: Tue Aug 30 09:20:38 2016 -0700 Hacks diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index c45e7a5..b56935b 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -40,7 +40,7 @@ # Below are what we set by default. May only work with SUN JVM. # For more on why as well as other possible settings, # see http://hbase.apache.org/book.html#performance -export HBASE_OPTS="-XX:+UseConcMarkSweepGC" +export HBASE_OPTS="-Djava.system.class.loader=org.apache.hadoop.hbase.RelocatingClassLoader -verbose:class -XX:+UseConcMarkSweepGC" # Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -XX:PermSize=128m -XX:MaxPermSize=128m" diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index b78f198..0e5a1ae 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -124,6 +124,10 @@ org.apache.hbase + hbase-shaded-libs + + + org.apache.hbase hbase-common diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 5b43553..5879174 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -229,6 +229,10 @@ org.apache.hbase + hbase-shaded-libs + + + org.apache.hbase hbase-protocol diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/RelocatingClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/RelocatingClassLoader.java new file mode 100644 index 0000000..bb010e1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/RelocatingClassLoader.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.net.URL; +import java.net.URLClassLoader; + +/** + * A class loader that rewrites a defined set of class names adding a relocation prefix so we we + * reference the shaded version of the class. Requires setting this class as the system class loader + * on the command line with this property -Djava.system.class.loader=THIS_CLASS. See + * http://docs.oracle.com/javase/8/docs/api/java/lang/ClassLoader.html#getSystemClassLoader-- + * We need to do ClassLoading and not delegate if we are to get a chance at intercepting all + * class loadings. If we delegate, we'll not get the opportunity to intercede because the parent + * will have managed the classloading for us. See + * http://stackoverflow.com/questions/25478223/why-is-custom-system-classloader-not-working + * + *

I can't have this class in hbase-shaded-libs, the natural place for it, because my + * PREFIX_TO_FIND String gets shaded and doesn't work anymore. + */ +public class RelocatingClassLoader extends URLClassLoader { + private static final String PREFIX_TO_FIND = "com.google.protobuf."; + private static final String PREFIX_TO_ADD = "org.apache.hadoop.hbase.shaded."; + + public RelocatingClassLoader(ClassLoader classLoader) { + // Pass in the CLASSPATH (URLs) that were in the parent so we can do as good as it. + // We are now the system class loader, not the passed in classloader. But we want its parent + // the bootstrap classloader to handle those classes we can't at; in particular the jdks. + super(getUrls(classLoader), classLoader.getParent()); + } + + private static URL[] getUrls(ClassLoader classLoader) { + URL [] urls = ((URLClassLoader)classLoader).getURLs(); + System.out.println(urls); + return urls; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + String str = name; + if (name.startsWith(PREFIX_TO_FIND)) { + str = PREFIX_TO_ADD + name; + System.out.println("RCL loading relocated " + name + " as \"" + str + "\""); + } + return super.loadClass(str); + } + + @Override + protected java.lang.Class loadClass(final String name, final boolean resolve) + throws ClassNotFoundException { + String str = name; + if (name.startsWith(PREFIX_TO_FIND)) { + str = PREFIX_TO_ADD + name; + System.out.println("RCL loading relocated " + name + " as \"" + str + "\""); + (new Throwable("Where am I?")).printStackTrace(System.out); + System.out.flush(); + } + Class clz = super.loadClass(str, resolve); + return clz; + } + + @Override + public URL getResource(String name) { + System.out.println("RCL getResource " + name); + return super.getResource(name); + } + + @Override + public URL findResource(String name) { + System.out.println("RCL findResource " + name); + return super.findResource(name); + } + + +} \ No newline at end of file diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 6cf1bb6..ef7caff 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -350,6 +350,10 @@ org.apache.hbase + hbase-shaded-libs + + + org.apache.hbase hbase-common diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java index 455cfd2..cdd77a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java @@ -62,4 +62,4 @@ public class ZKServerTool { System.out.println("ZK host: " + server.getHostname()); } } -} +} \ No newline at end of file diff --git a/hbase-shaded-libs/README.txt b/hbase-shaded-libs/README.txt new file mode 100644 index 0000000..4be401b --- /dev/null +++ b/hbase-shaded-libs/README.txt @@ -0,0 +1,33 @@ +These are the protobuf definition files used by hbase. ALL protobuf proto files +must live in this module whether test or spark or coprocessor endpoint protos +because we are being careful about what we expose of protobuf to downstreamers; +we are shading our version of protobuf so we can freely change it as needed. + +The produced java classes are generated into +src/main/java/org/apache/hadoop/hbase/protobuf/generated +and then checked in. The reasoning is that they change infrequently. + +To regenerate the classes after making definition file changes, ensure first that +the protobuf protoc tool is in your $PATH. You may need to download it and build +it first; its part of the protobuf package. For example, if using v2.5.0 of +protobuf, it is obtainable from here: + + https://github.com/google/protobuf/releases/tag/v2.5.0 + +HBase uses hadoop-maven-plugins:protoc goal to invoke the protoc command. You can +compile the protoc definitions by invoking maven with profile compile-protobuf or +passing in compile-protobuf property. + +mvn compile -Dcompile-protobuf +or +mvn compile -Pcompile-protobuf + +You may also want to define protoc.path for the protoc binary + +mvn compile -Dcompile-protobuf -Dprotoc.path=/opt/local/bin/protoc + +If you have added a new proto file, you should add it to the pom.xml file first. +Other modules also support the maven profile. + +After you've done the above, check it in and then check it in (or post a patch +on a JIRA with your definition file changes and the generated files). diff --git a/hbase-shaded-libs/pom.xml b/hbase-shaded-libs/pom.xml new file mode 100644 index 0000000..7904c3c --- /dev/null +++ b/hbase-shaded-libs/pom.xml @@ -0,0 +1,195 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 2.0.0-SNAPSHOT + .. + + + hbase-shaded-libs + Apache HBase - Shaded Libraries + Produces and hosts shaded versions of libraries + + + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + package + + shade + + + + + com.google.protobuf + org.apache.hadoop.hbase.shaded.com.google.protobuf + + + + + + commons-logging:commons-logging + com.github.stephenc.findbugs:findbugs-annotations + log4j:log4j + org.hamcrest:hamcrest-core + org.mockito:mockito-all + junit:junit + org.apache.hbase:hbase-annotations + + + + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + ${maven.assembly.version} + + true + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.hadoop + hadoop-maven-plugins + [2.0.5-alpha,) + + protoc + + + + + + + + + + + + + + + + + + + org.apache.hbase + hbase-annotations + + + jdk.tools + jdk.tools + + + + + + com.google.protobuf + protobuf-java + 2.5.0 + + + commons-logging + commons-logging + + + + + + + skip-rpc-tests + + + skip-rpc-tests + + + + true + + + + diff --git a/pom.xml b/pom.xml index b26c015..762f540 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ hbase-rest hbase-checkstyle hbase-external-blockcache + hbase-shaded-libs hbase-shaded hbase-spark hbase-archetypes @@ -1300,6 +1301,11 @@ org.apache.hbase + hbase-shaded-libs + ${project.version} + + + org.apache.hbase hbase-common ${project.version}