From 3cc346d7564eee7bd368c8cb983db5baf6f3d6a5 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 9 Apr 2018 13:37:44 -0500 Subject: [PATCH 1/4] HBASE-20332 shaded mapreduce module shouldn't include hadoop * modify the jar checking script to take args; make hadoop stuff optional * separate out checking the artifacts that have hadoop vs those that don't. * * Unfortunately means we need two modules for checking things * * put in a safety check that the support script for checking jar contents is maintained in both modules * * have to carve out an exception for o.a.hadoop.metrics2. :( * fix duplicated class warning * clean up dependencies in hbase-server and some modules that depend on it. * allow Hadoop to have its own htrace where it needs it * add a precommit check to make sure we're not using old htrace imports --- hbase-backup/pom.xml | 12 +- .../src/main/resources/hbase/checkstyle.xml | 4 +- hbase-client/pom.xml | 4 - hbase-common/pom.xml | 12 -- hbase-endpoint/pom.xml | 14 -- hbase-examples/pom.xml | 12 -- hbase-external-blockcache/pom.xml | 4 - hbase-hadoop2-compat/pom.xml | 6 - hbase-it/pom.xml | 6 - hbase-mapreduce/pom.xml | 30 +-- hbase-replication/pom.xml | 4 - hbase-rest/pom.xml | 19 +- hbase-rsgroup/pom.xml | 4 - hbase-server/pom.xml | 103 +++++----- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 54 ++++-- .../resources/ensure-jars-have-correct-contents.sh | 92 +++++++-- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++++++++++++++++- .../pom.xml | 215 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 129 +++++++++++++ hbase-shaded/pom.xml | 13 ++ hbase-shell/pom.xml | 14 -- hbase-testing-util/pom.xml | 16 -- hbase-thrift/pom.xml | 16 -- pom.xml | 65 +++---- 24 files changed, 773 insertions(+), 265 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 7afd51eaf0..00a996f51e 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -154,10 +154,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -264,9 +260,6 @@ 3.0 - - 3.0-SNAPSHOT - org.apache.hadoop @@ -276,6 +269,11 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml index 7ad797cf1f..148e2564e2 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml @@ -86,8 +86,10 @@ org.apache.commons.lang, org.apache.curator.shaded, org.apache.hadoop.classification, + org.apache.htrace, org.apache.htrace.shaded, - org.codehaus.jackson"/> + org.codehaus.jackson, + org.htrace"/> diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index f6247e30e3..bb99eec4ea 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -234,10 +234,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 5ae8e0b637..0aaccb89e3 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -314,12 +314,6 @@ hadoop-common - - - org.apache.htrace - htrace-core - - @@ -366,12 +360,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml index e9a8cf7132..a831d3a398 100644 --- a/hbase-endpoint/pom.xml +++ b/hbase-endpoint/pom.xml @@ -260,12 +260,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -296,10 +290,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -343,10 +333,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index c74c1ba24e..e417b2d212 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -232,12 +232,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - @@ -287,12 +281,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index 24c33ebe48..2479b46039 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -224,10 +224,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index ab33c72eb8..c1e42129da 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -167,12 +167,6 @@ limitations under the License. org.apache.hadoop hadoop-common ${hadoop.version} - - - org.apache.htrace - htrace-core - - org.apache.commons diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 9b1a167155..4c17245a9e 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -335,12 +335,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index af80737644..2bf693bc3d 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -196,6 +196,15 @@ org.apache.hbase hbase-server + + + + commons-logging + commons-logging + + org.apache.hbase @@ -246,10 +255,19 @@ junit test + com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + @@ -284,10 +302,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -334,10 +348,6 @@ org.apache.hadoop hadoop-hdfs - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -377,10 +387,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - org.apache.zookeeper zookeeper diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index d05c60ef64..b999c1d6cc 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -155,10 +155,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 617f254a2f..d06feec283 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -299,6 +299,19 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + + org.codehaus.jettison + jettison + + + stax + stax-api + + + org.glassfish.web @@ -374,12 +387,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml index 2d9a10d16c..1cc38549b9 100644 --- a/hbase-rsgroup/pom.xml +++ b/hbase-rsgroup/pom.xml @@ -198,10 +198,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 11361d886b..e9daf93aae 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -71,6 +71,14 @@ org.apache.maven.plugins maven-remote-resources-plugin 1.5 + + + + org.apache.hbase + hbase-resource-bundle + ${project.version} + + default @@ -392,12 +400,6 @@ org.apache.hbase hbase-metrics - - - org.apache.hbase - hbase-resource-bundle - true - commons-codec commons-codec @@ -437,19 +439,16 @@ jetty-webapp - + org.glassfish.web javax.servlet.jsp + - org.codehaus.jettison - jettison - - - stax - stax-api - - + javax.servlet.jsp + javax.servlet.jsp-api @@ -501,9 +500,20 @@ javax.servlet javax.servlet-api + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + - javax.ws.rs - javax.ws.rs-api + com.fasterxml.jackson.core + jackson-annotations @@ -511,11 +521,6 @@ org.apache.htrace htrace-core4 - - org.apache.htrace - htrace-core - ${htrace-hadoop.version} - com.lmax disruptor @@ -556,6 +561,15 @@ httpcore test + + + commons-logging + commons-logging + compile + org.apache.commons commons-crypto @@ -674,34 +688,10 @@ - - org.apache.hadoop - hadoop-distcp - ${hadoop-two.version} - org.apache.hadoop hadoop-common - - org.apache.hadoop - hadoop-auth - - - org.apache.hadoop - hadoop-annotations - ${hadoop-two.version} - - - org.apache.hadoop - hadoop-client - - - com.google.guava - guava - - - org.apache.hadoop hadoop-mapreduce-client-core @@ -796,21 +786,32 @@ org.apache.hadoop - hadoop-distcp - ${hadoop-three.version} + hadoop-common org.apache.hadoop - hadoop-common + hadoop-hdfs + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core org.apache.hadoop hadoop-hdfs + test-jar + test org.apache.hadoop - hadoop-annotations - ${hadoop-three.version} + hadoop-mapreduce-client-jobclient + test-jar + test org.apache.hadoop diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7322769f0b..7ba4a41782 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -26,7 +26,7 @@ Enforces our invariants for our shaded artifacts. e.g. shaded clients have a specific set of transitive dependencies and shaded clients only contain classes that are in particular packages. Does the enforcement through - the maven-enforcer-plugin and and integration test. + the maven-enforcer-plugin and integration test. Apache HBase Shaded Packaging Invariants @@ -34,11 +34,15 @@ - - org.apache.hbase - hbase-shaded-client - ${project.version} - + org.apache.hbase hbase-shaded-mapreduce @@ -113,6 +117,8 @@ com.github.stephenc.findbugs:* org.apache.htrace:* + + org.apache.hadoop:* @@ -158,18 +164,37 @@ - org.codehaus.mojo exec-maven-plugin 1.6.0 + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + check-jar-contents integration-test @@ -180,6 +205,9 @@ ${shell-executable} ${project.build.testOutputDirectory} false + ensure-jars-have-correct-contents.sh ${hbase-client-artifacts} diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 8bda8ce953..eff1d20302 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -15,33 +15,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar] -# -# accepts a single command line argument with a colon separated list of -# paths to jars to check. Iterates through each such passed jar and checks -# all the contained paths to make sure they follow the below constructed -# safe list. +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") -# we have to allow the directories that lead to the org/apache/hadoop dir -allowed_expr="(^org/$|^org/apache/$" +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" # We allow the following things to exist in our client artifacts: -# * classes in packages that start with org.apache.hadoop, which by -# convention should be in a path that looks like org/apache/hadoop -allowed_expr+="|^org/apache/hadoop/" +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" # * classes in packages that start with org.apache.hbase allowed_expr+="|^org/apache/hbase/" # * whatever in the "META-INF" directory allowed_expr+="|^META-INF/" # * the folding tables from jcodings allowed_expr+="|^tables/" -# * Hadoop's and HBase's default configuration files, which have the form +# * HBase's default configuration files, which have the form # "_module_-default.xml" -allowed_expr+="|^[^-]*-default.xml$" -# * Hadoop's versioning properties files, which have the form -# "_module_-version-info.properties" -allowed_expr+="|^[^-]*-version-info.properties$" -# * Hadoop's application classloader properties file. -allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +allowed_expr+="|^hbase-default.xml$" # public suffix list used by httpcomponents allowed_expr+="|^mozilla/$" allowed_expr+="|^mozilla/public-suffix-list.txt$" @@ -51,12 +85,30 @@ allowed_expr+="|^properties.dtd$" allowed_expr+="|^PropertyList-1.0.dtd$" +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + allowed_expr+=")" declare -i bad_artifacts=0 declare -a bad_contents -IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") for artifact in "${artifact_list[@]}"; do - bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) if [ ${#bad_contents[@]} -gt 0 ]; then echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" echo " Please check the following and either correct the build or update" diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml index cfcc357877..edc80cd151 100644 --- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml +++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml @@ -62,6 +62,10 @@ + org.apache.hbase hbase-mapreduce @@ -137,10 +141,6 @@ org.eclipse.jetty jetty-webapp - - org.glassfish.web - javax.servlet.jsp - org.glassfish.jersey.core jersey-server @@ -149,6 +149,17 @@ org.glassfish.jersey.containers jersey-container-servlet-core + + + org.glassfish.web + javax.servlet.jsp + + + javax.servlet.jsp + javax.servlet.jsp-api + @@ -158,12 +169,175 @@ release - - org.apache.maven.plugins - maven-shade-plugin - + + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + org.apache.hadoop:* + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + + + + + hadoop-2.0 + + + + !hadoop.profile + + + + + org.apache.hadoop + hadoop-common + provided + + + net.java.dev.jets3t + jets3t + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + javax.servlet + servlet-api + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + com.google.code.findbugs + jsr305 + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + javax.servlet.jsp + jsp-api + + + javax.servlet + servlet-api + + + io.netty + netty + + + stax + stax-api + + + xerces + xercesImpl + + + ${hadoop-two.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + com.google.guava + guava + + + + + org.apache.hadoop + hadoop-auth + provided + + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + ${hadoop-three.version} + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-auth + provided + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml new file mode 100644 index 0000000000..07789f4712 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml @@ -0,0 +1,215 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-with-hadoop-check-invariants + pom + + + Enforces our invariants for our shaded artifacts. e.g. shaded clients have + a specific set of transitive dependencies and shaded clients only contain + classes that are in particular packages. Does the enforcement through + the maven-enforcer-plugin and integration test. + + Apache HBase Shaded Packaging Invariants (with Hadoop bundled) + + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-core + provided + + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-6 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + + + check-jar-contents-for-stuff-with-hadoop + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + --allow-hadoop + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000000..eff1d20302 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") + +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" +# * classes in packages that start with org.apache.hbase +allowed_expr+="|^org/apache/hbase/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" +# * HBase's default configuration files, which have the form +# "_module_-default.xml" +allowed_expr+="|^hbase-default.xml$" +# public suffix list used by httpcomponents +allowed_expr+="|^mozilla/$" +allowed_expr+="|^mozilla/public-suffix-list.txt$" +# Comes from commons-configuration, not sure if relocatable. +allowed_expr+="|^digesterRules.xml$" +allowed_expr+="|^properties.dtd$" +allowed_expr+="|^PropertyList-1.0.dtd$" + + +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'" + fi +done + +# if there was atleast one bad artifact, exit with failure +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 24c515844e..93b122fe08 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -42,6 +42,7 @@ hbase-shaded-client hbase-shaded-mapreduce hbase-shaded-check-invariants + hbase-shaded-with-hadoop-check-invariants @@ -118,6 +119,7 @@ maven-shade-plugin + aggregate-into-a-jar-with-relocated-third-parties package shade @@ -449,12 +451,23 @@ META-INF/ECLIPSEF.RSA + + + commons-beanutils:commons-beanutils-core + + org/apache/commons/collections/*.class + + org.apache.hbase:hbase-server hbase-webapps/* hbase-webapps/**/* + **/*_jsp.class diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 8eaefaab83..85f0415605 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -277,12 +277,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -341,10 +335,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -408,10 +398,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml index 0f1b86cf83..032de61e3d 100644 --- a/hbase-testing-util/pom.xml +++ b/hbase-testing-util/pom.xml @@ -140,12 +140,6 @@ org.apache.hadoop hadoop-common compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -201,10 +195,6 @@ hadoop-minicluster compile - - org.apache.htrace - htrace-core - com.google.guava guava @@ -242,12 +232,6 @@ org.apache.hadoop hadoop-minicluster compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index 0142ccdb59..aec3cb4668 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -498,22 +498,12 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -571,12 +561,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/pom.xml b/pom.xml index ed7a1722ba..a943125475 100755 --- a/pom.xml +++ b/pom.xml @@ -1448,8 +1448,11 @@ 11.0.2 hbase-hadoop2-compat src/main/assembly/hadoop-two-compat.xml - 0.5.0 + + 3.6.2.Final + 0.5.0 1.7.7 1.10 @@ -1477,7 +1480,6 @@ 4.12 1.3 4.2.0-incubating - 3.2.0-incubating 1.2.17 2.1.0 @@ -1605,7 +1607,8 @@ org.mortbay.jetty:servlet-api, javax.servlet:servlet-api: These are excluded because they are the same implementations. I chose org.mortbay.jetty:servlet-api-2.5 instead, which is a third implementation of the same, because Hadoop also uses this version - javax.servlet:jsp-api in favour of org.mortbay.jetty:jsp-api-2.1 + javax.servlet:jsp-api in favour of javax.servlet.jsp:javax.servlet.jsp-api:2.3.1 since it + is what glassfish's jspC jar uses and that's where we get our own need for a jsp-api. --> @@ -1920,6 +1923,14 @@ commons-math3 ${commons-math.version} + + + commons-logging + commons-logging + 1.2 + org.apache.zookeeper zookeeper @@ -1983,6 +1994,16 @@ jackson-jaxrs-json-provider ${jackson.version} + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + com.fasterxml.jackson.core jackson-databind @@ -2086,6 +2107,12 @@ javax.servlet.jsp ${glassfish.jsp.version} + + + javax.servlet.jsp + javax.servlet.jsp-api + 2.3.1 + org.glassfish javax.el @@ -2550,10 +2577,6 @@ hadoop-hdfs ${hadoop-two.version} - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2595,10 +2618,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2643,10 +2662,6 @@ hadoop-common ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-beanutils commons-beanutils @@ -2697,10 +2712,6 @@ hadoop-minicluster ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient @@ -2819,10 +2830,6 @@ hadoop-hdfs ${hadoop-three.version} - - org.apache.htrace - htrace-core - com.sun.jersey jersey-core @@ -2868,10 +2875,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2952,10 +2955,6 @@ com.sun.jersey jersey-server - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -3020,10 +3019,6 @@ hadoop-minicluster ${hadoop-three.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient -- 2.16.1 From 1e09eb4cecc2940588f97a8a790b534e06269329 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 24 Apr 2018 14:51:12 -0500 Subject: [PATCH 2/4] HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs. --- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 5 ++ .../hbase-shaded-client-byo-hadoop/pom.xml | 70 ++++++++++++++++++++++ hbase-shaded/hbase-shaded-client/pom.xml | 35 +++++++++-- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++-------- hbase-shaded/pom.xml | 6 ++ 5 files changed, 115 insertions(+), 31 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7ba4a41782..287a986568 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -48,6 +48,11 @@ hbase-shaded-mapreduce ${project.version} + + org.apache.hbase + hbase-shaded-client-byo-hadoop + ${project.version} + com.github.stephenc.findbugs diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml new file mode 100644 index 0000000000..c51a1af745 --- /dev/null +++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + hbase-shaded + org.apache.hbase + 3.0.0-SNAPSHOT + .. + + hbase-shaded-client-byo-hadoop + Apache HBase - Shaded - Client + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + maven-assembly-plugin + + true + + + + + + + org.apache.hbase + hbase-client + + + + + + release + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + + diff --git a/hbase-shaded/hbase-shaded-client/pom.xml b/hbase-shaded/hbase-shaded-client/pom.xml index 72a5b6058e..5ac3ef5adb 100644 --- a/hbase-shaded/hbase-shaded-client/pom.xml +++ b/hbase-shaded/hbase-shaded-client/pom.xml @@ -28,7 +28,7 @@ .. hbase-shaded-client - Apache HBase - Shaded - Client + Apache HBase - Shaded - Client (with Hadoop bundled) @@ -51,6 +51,7 @@ org.apache.hbase hbase-client + ${project.version} @@ -59,10 +60,34 @@ release - - org.apache.maven.plugins - maven-shade-plugin - + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml index edc80cd151..598f3af938 100644 --- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml +++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml @@ -169,32 +169,10 @@ release - - - org.apache.maven.plugins - maven-shade-plugin - - - aggregate-into-a-jar-with-relocated-third-parties - - - - org.apache.hadoop:* - - org.apache.hbase:hbase-resource-bundle - org.slf4j:* - com.google.code.findbugs:* - com.github.stephenc.findbugs:* - org.apache.htrace:* - org.apache.yetus:* - log4j:* - commons-logging:* - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 93b122fe08..9eb30e0941 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -39,6 +39,7 @@ org.apache.hadoop.hbase.shaded + hbase-shaded-client-byo-hadoop hbase-shaded-client hbase-shaded-mapreduce hbase-shaded-check-invariants @@ -131,6 +132,11 @@ false + + org.apache.hadoop:* + org.apache.hbase:hbase-resource-bundle org.slf4j:* com.google.code.findbugs:* -- 2.16.1 From 9eca9005e121d2105cdc6159921bc7eb9b0eecee Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Wed, 7 Feb 2018 18:37:39 -0500 Subject: [PATCH 3/4] HBASE-19735 Create a client-tarball assembly Provides an extra client descriptor to build a second tarball with a reduced set of dependencies. Not of great impact now, but will build the way for better in the future. Signed-off-by: Sean Busbey Conflicts: hbase-assembly/pom.xml --- hbase-assembly/pom.xml | 33 +++-- .../src/main/assembly/client-components.xml | 92 ++++++++++++++ hbase-assembly/src/main/assembly/client.xml | 137 +++++++++++++++++++++ hbase-procedure/pom.xml | 4 +- hbase-spark/pom.xml | 6 + pom.xml | 10 ++ 6 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 hbase-assembly/src/main/assembly/client-components.xml create mode 100644 hbase-assembly/src/main/assembly/client.xml diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 5da105b24e..4fa859ac99 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -95,6 +95,7 @@ gnu ${assembly.file} + src/main/assembly/client.xml @@ -220,22 +221,22 @@ test - org.apache.hbase - hbase-hadoop-compat - - - com.google.guava - guava - - + org.apache.hbase + hbase-hadoop-compat + + + com.google.guava + guava + + - org.apache.hbase - ${compat.module} + org.apache.hbase + ${compat.module} - org.apache.hbase - hbase-shell + org.apache.hbase + hbase-shell org.apache.hbase @@ -315,6 +316,14 @@ jline jline + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml new file mode 100644 index 0000000000..2fd1b579e2 --- /dev/null +++ b/hbase-assembly/src/main/assembly/client-components.xml @@ -0,0 +1,92 @@ + + + + + + + + + ${project.basedir}/../target/site + docs + + + + ${project.basedir}/.. + . + + CHANGES.txt + README.txt + + 0644 + + + + ${project.basedir}/../conf + conf + 0644 + 0755 + + + + + ${project.basedir}/../bin + bin + + get-active-master.rb + hbase + hbase-common.sh + hbase-config.sh + hbase-jruby + hirb.rb + + + 0755 + 0755 + + + + ${project.basedir}/../bin + bin + + hbase.cmd + hbase-config.cmd + + + + + ${project.basedir}/../hbase-shell/src/main/ruby + lib/ruby + 0644 + 0755 + + + + ${project.basedir}/../hbase-server/target/native + lib/native + 0755 + 0755 + + *.so + *.dylib + + + + diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml new file mode 100644 index 0000000000..79519613c3 --- /dev/null +++ b/hbase-assembly/src/main/assembly/client.xml @@ -0,0 +1,137 @@ + + + + + + client-bin + + tar.gz + + + hbase-${project.version}-client + + src/main/assembly/client-components.xml + + + + true + + + org.apache.hbase:hbase-annotations + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-hadoop2-compat + org.apache.hbase:hbase-mapreduce + org.apache.hbase:hbase-metrics + org.apache.hbase:hbase-metrics-api + org.apache.hbase:hbase-procedure + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-server + org.apache.hbase:hbase-zookeeper + + + + lib + false + + + + + com.sun.jersey:* + com.sun.jersey.contribs:* + + org.apache.hbase:hbase-external-blockcache + org.apache.hbase:hbase-http + org.apache.hbase:hbase-replication + org.apache.hbase:hbase-rest + org.apache.hbase:hbase-rsgroup + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + org.apache.hbase:hbase-shell + org.apache.hbase:hbase-thrift + org.jruby:jruby-complete + + + + + + + true + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + + + shaded-lib + false + + + + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + false + + + + + + + + + ${project.build.directory}/maven-shared-archive-resources/META-INF/LICENSE + . + LICENSE.txt + unix + + + ${project.build.directory}/NOTICE.aggregate + . + NOTICE.txt + unix + + + ${basedir}/src/main/resources/META-INF/LEGAL + . + LEGAL + unix + + + + + + + lib/ruby + + org.jruby:jruby-complete + + + + + diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index 3fedda8e19..b89acdb8a6 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -57,8 +57,8 @@ org.apache.hbase hbase-common - ${project.version} - tests + test-jar + test org.apache.hbase diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml index 7654be49bd..a5f96b4b46 100644 --- a/hbase-spark/pom.xml +++ b/hbase-spark/pom.xml @@ -236,6 +236,12 @@ hbase-common ${project.version} + + org.apache.hbase + hbase-common + test-jar + test + org.apache.hbase hbase-annotations diff --git a/pom.xml b/pom.xml index a943125475..b9a1b7725e 100755 --- a/pom.xml +++ b/pom.xml @@ -1853,6 +1853,16 @@ test-jar test + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-mapreduce + ${project.version} + com.github.stephenc.findbugs -- 2.16.1 From a7706ea09659fb74702f961def130e682c317c68 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Fri, 18 May 2018 11:11:42 -0500 Subject: [PATCH 4/4] HBASE-20615 emphasize shaded artifacts in client tarball. --- bin/hbase | 226 ++++++++++++++++++--- bin/hbase-config.sh | 10 + hbase-assembly/pom.xml | 21 +- .../src/main/assembly/client-components.xml | 43 +++- hbase-assembly/src/main/assembly/client.xml | 131 ++++++------ hbase-assembly/src/main/assembly/components.xml | 3 +- .../src/main/assembly/hadoop-two-compat.xml | 80 +++++++- pom.xml | 6 + 8 files changed, 417 insertions(+), 103 deletions(-) diff --git a/bin/hbase b/bin/hbase index 4f1c854dfa..559a02e9f8 100755 --- a/bin/hbase +++ b/bin/hbase @@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then in_dev_env=true fi +# Detect if we are in the omnibus tarball +in_omnibus_tarball="false" +if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then + in_omnibus_tarball="true" +fi + read -d '' options_string << EOF Options: - --config DIR Configuration direction to use. Default: ./conf - --hosts HOSTS Override the list in 'regionservers' file - --auth-as-server Authenticate to ZooKeeper using servers configuration + --config DIR Configuration direction to use. Default: ./conf + --hosts HOSTS Override the list in 'regionservers' file + --auth-as-server Authenticate to ZooKeeper using servers configuration + --internal-classpath Skip attempting to use client facing jars (WARNING: unstable results between versions) EOF # if no args specified, show usage if [ $# = 0 ]; then @@ -87,16 +94,18 @@ if [ $# = 0 ]; then echo " shell Run the HBase shell" echo " hbck Run the hbase 'fsck' tool" echo " snapshot Tool for managing snapshots" - echo " wal Write-ahead-log analyzer" - echo " hfile Store file analyzer" - echo " zkcli Run the ZooKeeper shell" - echo " master Run an HBase HMaster node" - echo " regionserver Run an HBase HRegionServer node" - echo " zookeeper Run a ZooKeeper server" - echo " rest Run an HBase REST server" - echo " thrift Run the HBase Thrift server" - echo " thrift2 Run the HBase Thrift2 server" - echo " clean Run the HBase clean up script" + if [ "${in_omnibus_tarball}" = "true" ]; then + echo " wal Write-ahead-log analyzer" + echo " hfile Store file analyzer" + echo " zkcli Run the ZooKeeper shell" + echo " master Run an HBase HMaster node" + echo " regionserver Run an HBase HRegionServer node" + echo " zookeeper Run a ZooKeeper server" + echo " rest Run an HBase REST server" + echo " thrift Run the HBase Thrift server" + echo " thrift2 Run the HBase Thrift2 server" + echo " clean Run the HBase clean up script" + fi echo " classpath Dump hbase CLASSPATH" echo " mapredcp Dump CLASSPATH entries required by mapreduce" echo " pe Run PerformanceEvaluation" @@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do fi done +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +# Allow this functionality to be disabled +if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then + HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) +fi + # Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; +declare shaded_jar + +if [ "${INTERNAL_CLASSPATH}" != "true" ]; then + # find our shaded jars + declare shaded_client + declare shaded_client_byo_hadoop + declare shaded_mapreduce + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do + if [[ "${f}" =~ byo-hadoop ]]; then + shaded_client_byo_hadoop="${f}" + else + shaded_client="${f}" + fi + done + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do + shaded_mapreduce="${f}" + done + + # If command can use our shaded client, use it + declare -a commands_in_client_jar=("classpath" "version") + for c in "${commands_in_client_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client_byo_hadoop}" + else + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client}" + fi + break + fi + done + + # If command needs our shaded mapreduce, use it + # N.B "mapredcp" is not included here because in the shaded case it skips our built classpath + declare -a commands_in_mr_jar=("hbck" "snapshot" "canary" "regionsplitter" "pre-upgrade") + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + + # Some commands specifically only can use shaded mapreduce when we'll get a full hadoop classpath at runtime + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + declare -a commands_in_mr_need_hadoop=("backup" "restore" "rowcounter" "cellcounter") + for c in "${commands_in_mr_need_hadoop[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + fi +fi + + +if [ -n "${shaded_jar}" ] && [ -f "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_jar}" +# fall through to grabbing all the lib jars and hope we're in the omnibus tarball +# +# N.B. shell specifically can't rely on the shaded artifacts because RSGroups is only +# available as non-shaded +# +# N.B. pe and ltt can't easily rely on shaded artifacts because they live in hbase-mapreduce:test-jar +# and need some other jars that haven't been relocated. Currently enumerating that list +# is too hard to be worth it. +# +else + for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + # make it easier to check for shaded/not later on. + shaded_jar="" +fi +for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + CLASSPATH="${CLASSPATH}:${f}" + fi done # default log directory & file @@ -201,9 +300,9 @@ fi function append_path() { if [ -z "$1" ]; then - echo $2 + echo "$2" else - echo $1:$2 + echo "$1:$2" fi } @@ -214,18 +313,34 @@ if [ "$HBASE_LIBRARY_PATH" != "" ]; then JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") fi -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -# Allow this functionality to be disabled -if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then - HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) - if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +#If configured and available, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" "${HADOOP_IN_PATH}" \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") fi + CLASSPATH=$(append_path "${CLASSPATH}" "$(${HADOOP_IN_PATH} classpath 2>/dev/null)") +else + # Otherwise, if we're providing Hadoop we should include htrace 3 if we were built with a version that needs it. + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core-3*.jar "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + # Some commands require special handling when using shaded jars. For these cases, we rely on hbase-shaded-mapreduce + # instead of hbase-shaded-client* because we make use of some IA.Private classes that aren't in the latter. However, + # we don't invoke them using the "hadoop jar" command so we need to ensure there are some Hadoop classes available + # when we're not doing runtime hadoop classpath lookup. + # + # luckily the set of classes we need are those packaged in the shaded-client. + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ] && [ -n "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_client:?We couldn\'t find the shaded client jar even though we did find the shaded MR jar. for command ${COMMAND} we need both. please use --internal-classpath as a workaround.}" + break + fi + done fi # Add user-specified CLASSPATH last @@ -250,11 +365,11 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` fi if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib") fi if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/lib/native/${JAVA_PLATFORM}") fi fi @@ -379,8 +494,30 @@ elif [ "$COMMAND" = "zkcli" ] ; then done elif [ "$COMMAND" = "backup" ] ; then CLASS='org.apache.hadoop.hbase.backup.BackupDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "restore" ] ; then CLASS='org.apache.hadoop.hbase.backup.RestoreDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + for f in "${HBASE_HOME}"/lib/commons-lang3*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "upgrade" ] ; then echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0." echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading." @@ -451,9 +588,24 @@ elif [ "$COMMAND" = "clean" ] ; then "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ exit $? elif [ "$COMMAND" = "mapredcp" ] ; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + if [ "${INTERNAL_CLASSPATH}" != "true" ] && [ -f "${shaded_jar}" ]; then + echo -n "${shaded_jar}" + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + echo -n ":${f}" + fi + done + echo "" + exit 0 + fi CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH + echo "$CLASSPATH" exit 0 elif [ "$COMMAND" = "pe" ] ; then CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' @@ -500,8 +652,20 @@ else fi HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" +# by now if we're running a command it means we need logging +for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi +done + # Exec unless HBASE_NOEXEC is set. export CLASSPATH +if [ "${DEBUG}" = "true" ]; then + echo "classpath=${CLASSPATH}" >&2 + HBASE_OPTS="${HBASE_OPTS} -Xdiag" +fi if [ "${HBASE_NOEXEC}" != "" ]; then "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh index 2e95ae71ad..1054751e63 100644 --- a/bin/hbase-config.sh +++ b/bin/hbase-config.sh @@ -84,6 +84,16 @@ do exit 1 fi shift + elif [ "--internal-classpath" = "$1" ] + then + shift + # shellcheck disable=SC2034 + INTERNAL_CLASSPATH="true" + elif [ "--debug" = "$1" ] + then + shift + # shellcheck disable=SC2034 + DEBUG="true" else # Presume we are at end of options and break break diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 4fa859ac99..fd1415fd20 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -190,6 +190,19 @@ + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-client-byo-hadoop + + + org.apache.hbase + hbase-shaded-mapreduce + org.apache.hbase @@ -316,14 +329,6 @@ jline jline - - org.apache.hbase - hbase-shaded-client - - - org.apache.hbase - hbase-shaded-mapreduce - diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml index 2fd1b579e2..2369f28d20 100644 --- a/hbase-assembly/src/main/assembly/client-components.xml +++ b/hbase-assembly/src/main/assembly/client-components.xml @@ -32,7 +32,8 @@ ${project.basedir}/.. . - CHANGES.txt + CHANGES.md + RELEASENOTES.md README.txt 0644 @@ -56,7 +57,6 @@ hbase-config.sh hbase-jruby hirb.rb - 0755 0755 @@ -88,5 +88,44 @@ *.dylib + + + + ${project.basedir}/../hbase-server/target/ + lib + + ${server.test.jar} + + 0644 + + + ${project.basedir}/../hbase-mapreduce/target/ + lib + + ${mapreduce.test.jar} + + 0644 + + + ${project.basedir}/../hbase-common/target/ + lib + + ${common.test.jar} + + 0644 + + + ${project.basedir}/../hbase-zookeeper/target/ + lib + + ${zookeeper.test.jar} + + 0644 + diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml index 79519613c3..c875b9534f 100644 --- a/hbase-assembly/src/main/assembly/client.xml +++ b/hbase-assembly/src/main/assembly/client.xml @@ -31,73 +31,37 @@ src/main/assembly/client-components.xml + true - - org.apache.hbase:hbase-annotations - org.apache.hbase:hbase-client - org.apache.hbase:hbase-common - org.apache.hbase:hbase-hadoop-compat - org.apache.hbase:hbase-hadoop2-compat - org.apache.hbase:hbase-mapreduce - org.apache.hbase:hbase-metrics - org.apache.hbase:hbase-metrics-api - org.apache.hbase:hbase-procedure - org.apache.hbase:hbase-protocol - org.apache.hbase:hbase-protocol-shaded - org.apache.hbase:hbase-server - org.apache.hbase:hbase-zookeeper + org.apache.hbase:hbase-shell - - lib false - - - + lib + + + + org.jruby:jruby-complete com.sun.jersey:* com.sun.jersey.contribs:* - - org.apache.hbase:hbase-external-blockcache - org.apache.hbase:hbase-http - org.apache.hbase:hbase-replication - org.apache.hbase:hbase-rest - org.apache.hbase:hbase-rsgroup - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - org.apache.hbase:hbase-shell - org.apache.hbase:hbase-thrift - org.jruby:jruby-complete + jline:jline + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-client-byo-hadoop + org.apache.hbase:hbase-shaded-mapreduce + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 - - - - - - true - - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - - - shaded-lib - false - - - - - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - false - - + + @@ -123,15 +87,66 @@ + + + lib/shaded-clients + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + - lib/ruby org.jruby:jruby-complete + + + lib/client-facing-thirdparty + true + + + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 59502d67b4..2eb16e7649 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -37,7 +37,8 @@ ${project.basedir}/.. . - CHANGES.txt + CHANGES.md + RELEASENOTES.md README.txt 0644 diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index 69a800bc2a..72437839df 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -66,18 +66,51 @@ lib false - + org.jruby:jruby-complete com.sun.jersey:* com.sun.jersey.contribs:* jline:jline + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-client-byo-hadoop + org.apache.hbase:hbase-shaded-mapreduce + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + + true + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + + lib/shaded-clients + false + + + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + + + + @@ -101,15 +134,56 @@ - + lib/ruby org.jruby:jruby-complete + + + lib/client-facing-thirdparty + true + + + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + lib/zkcli diff --git a/pom.xml b/pom.xml index b9a1b7725e..ea02fe8c3c 100755 --- a/pom.xml +++ b/pom.xml @@ -1555,6 +1555,7 @@ hbase-annotations-${project.version}-tests.jar hbase-rsgroup-${project.version}-tests.jar hbase-mapreduce-${project.version}-tests.jar + hbase-zookeeper-${project.version}-tests.jar bash 2.21.0 surefire-junit47 @@ -1858,6 +1859,11 @@ hbase-shaded-client ${project.version} + + org.apache.hbase + hbase-shaded-client-byo-hadoop + ${project.version} + org.apache.hbase hbase-shaded-mapreduce -- 2.16.1