From 93e9ee22de2eb53ba976ecf2aa7e1c6887ad1a9b Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 9 Apr 2018 13:37:44 -0500 Subject: [PATCH] HBASE-20332 shaded mapreduce module shouldn't include hadoop * modify the jar checking script to take args; make hadoop stuff optional * separate out checking the artifacts that have hadoop vs those that don't. * * Unfortunately means we need two modules for checking things * * put in a safety check that the support script for checking jar contents is maintained in both modules * * have to carve out an exception for o.a.hadoop.metrics2. :( * fix duplicated class warning * clean up dependencies in hbase-server and some modules that depend on it. * allow Hadoop to have its own htrace where it needs it * add a precommit check to make sure we're not using old htrace imports --- hbase-backup/pom.xml | 12 +- .../resources/hbase/checkstyle-suppressions.xml | 1 + .../src/main/resources/hbase/checkstyle.xml | 4 +- hbase-client/pom.xml | 4 - hbase-common/pom.xml | 12 -- hbase-endpoint/pom.xml | 14 -- hbase-examples/pom.xml | 12 -- hbase-external-blockcache/pom.xml | 4 - hbase-hadoop2-compat/pom.xml | 6 - hbase-it/pom.xml | 6 - hbase-mapreduce/pom.xml | 30 +-- hbase-replication/pom.xml | 4 - hbase-rest/pom.xml | 19 +- hbase-rsgroup/pom.xml | 4 - hbase-server/pom.xml | 103 +++++----- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 54 ++++-- .../resources/ensure-jars-have-correct-contents.sh | 92 +++++++-- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++++++++++++++++- .../pom.xml | 215 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 129 +++++++++++++ hbase-shaded/pom.xml | 13 ++ hbase-shell/pom.xml | 14 -- hbase-testing-util/pom.xml | 16 -- hbase-thrift/pom.xml | 16 -- pom.xml | 65 +++---- 25 files changed, 774 insertions(+), 265 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 7afd51eaf0..00a996f51e 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -154,10 +154,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -264,9 +260,6 @@ 3.0 - - 3.0-SNAPSHOT - org.apache.hadoop @@ -276,6 +269,11 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml index ad79163c55..464f455451 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml @@ -39,4 +39,5 @@ + diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml index 7ad797cf1f..148e2564e2 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml @@ -86,8 +86,10 @@ org.apache.commons.lang, org.apache.curator.shaded, org.apache.hadoop.classification, + org.apache.htrace, org.apache.htrace.shaded, - org.codehaus.jackson"/> + org.codehaus.jackson, + org.htrace"/> diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index f6247e30e3..bb99eec4ea 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -234,10 +234,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 5ae8e0b637..0aaccb89e3 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -314,12 +314,6 @@ hadoop-common - - - org.apache.htrace - htrace-core - - @@ -366,12 +360,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml index e9a8cf7132..a831d3a398 100644 --- a/hbase-endpoint/pom.xml +++ b/hbase-endpoint/pom.xml @@ -260,12 +260,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -296,10 +290,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -343,10 +333,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index c74c1ba24e..e417b2d212 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -232,12 +232,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - @@ -287,12 +281,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index 24c33ebe48..2479b46039 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -224,10 +224,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index ab33c72eb8..c1e42129da 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -167,12 +167,6 @@ limitations under the License. org.apache.hadoop hadoop-common ${hadoop.version} - - - org.apache.htrace - htrace-core - - org.apache.commons diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 9b1a167155..4c17245a9e 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -335,12 +335,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index af80737644..2bf693bc3d 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -196,6 +196,15 @@ org.apache.hbase hbase-server + + + + commons-logging + commons-logging + + org.apache.hbase @@ -246,10 +255,19 @@ junit test + com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + @@ -284,10 +302,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -334,10 +348,6 @@ org.apache.hadoop hadoop-hdfs - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -377,10 +387,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - org.apache.zookeeper zookeeper diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index d05c60ef64..b999c1d6cc 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -155,10 +155,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 617f254a2f..d06feec283 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -299,6 +299,19 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + + org.codehaus.jettison + jettison + + + stax + stax-api + + + org.glassfish.web @@ -374,12 +387,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml index 2d9a10d16c..1cc38549b9 100644 --- a/hbase-rsgroup/pom.xml +++ b/hbase-rsgroup/pom.xml @@ -198,10 +198,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 11361d886b..e9daf93aae 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -71,6 +71,14 @@ org.apache.maven.plugins maven-remote-resources-plugin 1.5 + + + + org.apache.hbase + hbase-resource-bundle + ${project.version} + + default @@ -392,12 +400,6 @@ org.apache.hbase hbase-metrics - - - org.apache.hbase - hbase-resource-bundle - true - commons-codec commons-codec @@ -437,19 +439,16 @@ jetty-webapp - + org.glassfish.web javax.servlet.jsp + - org.codehaus.jettison - jettison - - - stax - stax-api - - + javax.servlet.jsp + javax.servlet.jsp-api @@ -501,9 +500,20 @@ javax.servlet javax.servlet-api + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + - javax.ws.rs - javax.ws.rs-api + com.fasterxml.jackson.core + jackson-annotations @@ -511,11 +521,6 @@ org.apache.htrace htrace-core4 - - org.apache.htrace - htrace-core - ${htrace-hadoop.version} - com.lmax disruptor @@ -556,6 +561,15 @@ httpcore test + + + commons-logging + commons-logging + compile + org.apache.commons commons-crypto @@ -674,34 +688,10 @@ - - org.apache.hadoop - hadoop-distcp - ${hadoop-two.version} - org.apache.hadoop hadoop-common - - org.apache.hadoop - hadoop-auth - - - org.apache.hadoop - hadoop-annotations - ${hadoop-two.version} - - - org.apache.hadoop - hadoop-client - - - com.google.guava - guava - - - org.apache.hadoop hadoop-mapreduce-client-core @@ -796,21 +786,32 @@ org.apache.hadoop - hadoop-distcp - ${hadoop-three.version} + hadoop-common org.apache.hadoop - hadoop-common + hadoop-hdfs + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core org.apache.hadoop hadoop-hdfs + test-jar + test org.apache.hadoop - hadoop-annotations - ${hadoop-three.version} + hadoop-mapreduce-client-jobclient + test-jar + test org.apache.hadoop diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7322769f0b..7ba4a41782 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -26,7 +26,7 @@ Enforces our invariants for our shaded artifacts. e.g. shaded clients have a specific set of transitive dependencies and shaded clients only contain classes that are in particular packages. Does the enforcement through - the maven-enforcer-plugin and and integration test. + the maven-enforcer-plugin and integration test. Apache HBase Shaded Packaging Invariants @@ -34,11 +34,15 @@ - - org.apache.hbase - hbase-shaded-client - ${project.version} - + org.apache.hbase hbase-shaded-mapreduce @@ -113,6 +117,8 @@ com.github.stephenc.findbugs:* org.apache.htrace:* + + org.apache.hadoop:* @@ -158,18 +164,37 @@ - org.codehaus.mojo exec-maven-plugin 1.6.0 + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + check-jar-contents integration-test @@ -180,6 +205,9 @@ ${shell-executable} ${project.build.testOutputDirectory} false + ensure-jars-have-correct-contents.sh ${hbase-client-artifacts} diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 8bda8ce953..eff1d20302 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -15,33 +15,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar] -# -# accepts a single command line argument with a colon separated list of -# paths to jars to check. Iterates through each such passed jar and checks -# all the contained paths to make sure they follow the below constructed -# safe list. +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") -# we have to allow the directories that lead to the org/apache/hadoop dir -allowed_expr="(^org/$|^org/apache/$" +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" # We allow the following things to exist in our client artifacts: -# * classes in packages that start with org.apache.hadoop, which by -# convention should be in a path that looks like org/apache/hadoop -allowed_expr+="|^org/apache/hadoop/" +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" # * classes in packages that start with org.apache.hbase allowed_expr+="|^org/apache/hbase/" # * whatever in the "META-INF" directory allowed_expr+="|^META-INF/" # * the folding tables from jcodings allowed_expr+="|^tables/" -# * Hadoop's and HBase's default configuration files, which have the form +# * HBase's default configuration files, which have the form # "_module_-default.xml" -allowed_expr+="|^[^-]*-default.xml$" -# * Hadoop's versioning properties files, which have the form -# "_module_-version-info.properties" -allowed_expr+="|^[^-]*-version-info.properties$" -# * Hadoop's application classloader properties file. -allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +allowed_expr+="|^hbase-default.xml$" # public suffix list used by httpcomponents allowed_expr+="|^mozilla/$" allowed_expr+="|^mozilla/public-suffix-list.txt$" @@ -51,12 +85,30 @@ allowed_expr+="|^properties.dtd$" allowed_expr+="|^PropertyList-1.0.dtd$" +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + allowed_expr+=")" declare -i bad_artifacts=0 declare -a bad_contents -IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") for artifact in "${artifact_list[@]}"; do - bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) if [ ${#bad_contents[@]} -gt 0 ]; then echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" echo " Please check the following and either correct the build or update" diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml index cfcc357877..edc80cd151 100644 --- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml +++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml @@ -62,6 +62,10 @@ + org.apache.hbase hbase-mapreduce @@ -137,10 +141,6 @@ org.eclipse.jetty jetty-webapp - - org.glassfish.web - javax.servlet.jsp - org.glassfish.jersey.core jersey-server @@ -149,6 +149,17 @@ org.glassfish.jersey.containers jersey-container-servlet-core + + + org.glassfish.web + javax.servlet.jsp + + + javax.servlet.jsp + javax.servlet.jsp-api + @@ -158,12 +169,175 @@ release - - org.apache.maven.plugins - maven-shade-plugin - + + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + org.apache.hadoop:* + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + + + + + hadoop-2.0 + + + + !hadoop.profile + + + + + org.apache.hadoop + hadoop-common + provided + + + net.java.dev.jets3t + jets3t + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + javax.servlet + servlet-api + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + com.google.code.findbugs + jsr305 + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + javax.servlet.jsp + jsp-api + + + javax.servlet + servlet-api + + + io.netty + netty + + + stax + stax-api + + + xerces + xercesImpl + + + ${hadoop-two.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + com.google.guava + guava + + + + + org.apache.hadoop + hadoop-auth + provided + + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + ${hadoop-three.version} + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-auth + provided + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml new file mode 100644 index 0000000000..07789f4712 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml @@ -0,0 +1,215 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-with-hadoop-check-invariants + pom + + + Enforces our invariants for our shaded artifacts. e.g. shaded clients have + a specific set of transitive dependencies and shaded clients only contain + classes that are in particular packages. Does the enforcement through + the maven-enforcer-plugin and integration test. + + Apache HBase Shaded Packaging Invariants (with Hadoop bundled) + + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-core + provided + + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-6 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + + + check-jar-contents-for-stuff-with-hadoop + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + --allow-hadoop + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000000..eff1d20302 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") + +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" +# * classes in packages that start with org.apache.hbase +allowed_expr+="|^org/apache/hbase/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" +# * HBase's default configuration files, which have the form +# "_module_-default.xml" +allowed_expr+="|^hbase-default.xml$" +# public suffix list used by httpcomponents +allowed_expr+="|^mozilla/$" +allowed_expr+="|^mozilla/public-suffix-list.txt$" +# Comes from commons-configuration, not sure if relocatable. +allowed_expr+="|^digesterRules.xml$" +allowed_expr+="|^properties.dtd$" +allowed_expr+="|^PropertyList-1.0.dtd$" + + +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'" + fi +done + +# if there was atleast one bad artifact, exit with failure +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 1efed1e938..36f5db397f 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -42,6 +42,7 @@ hbase-shaded-client hbase-shaded-mapreduce hbase-shaded-check-invariants + hbase-shaded-with-hadoop-check-invariants @@ -118,6 +119,7 @@ maven-shade-plugin + aggregate-into-a-jar-with-relocated-third-parties package shade @@ -447,12 +449,23 @@ META-INF/ECLIPSEF.RSA + + + commons-beanutils:commons-beanutils-core + + org/apache/commons/collections/*.class + + org.apache.hbase:hbase-server hbase-webapps/* hbase-webapps/**/* + **/*_jsp.class diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 8eaefaab83..85f0415605 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -277,12 +277,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -341,10 +335,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -408,10 +398,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml index 0f1b86cf83..032de61e3d 100644 --- a/hbase-testing-util/pom.xml +++ b/hbase-testing-util/pom.xml @@ -140,12 +140,6 @@ org.apache.hadoop hadoop-common compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -201,10 +195,6 @@ hadoop-minicluster compile - - org.apache.htrace - htrace-core - com.google.guava guava @@ -242,12 +232,6 @@ org.apache.hadoop hadoop-minicluster compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index 814650d932..990e41c003 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -503,22 +503,12 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -576,12 +566,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/pom.xml b/pom.xml index ed7a1722ba..a943125475 100755 --- a/pom.xml +++ b/pom.xml @@ -1448,8 +1448,11 @@ 11.0.2 hbase-hadoop2-compat src/main/assembly/hadoop-two-compat.xml - 0.5.0 + + 3.6.2.Final + 0.5.0 1.7.7 1.10 @@ -1477,7 +1480,6 @@ 4.12 1.3 4.2.0-incubating - 3.2.0-incubating 1.2.17 2.1.0 @@ -1605,7 +1607,8 @@ org.mortbay.jetty:servlet-api, javax.servlet:servlet-api: These are excluded because they are the same implementations. I chose org.mortbay.jetty:servlet-api-2.5 instead, which is a third implementation of the same, because Hadoop also uses this version - javax.servlet:jsp-api in favour of org.mortbay.jetty:jsp-api-2.1 + javax.servlet:jsp-api in favour of javax.servlet.jsp:javax.servlet.jsp-api:2.3.1 since it + is what glassfish's jspC jar uses and that's where we get our own need for a jsp-api. --> @@ -1920,6 +1923,14 @@ commons-math3 ${commons-math.version} + + + commons-logging + commons-logging + 1.2 + org.apache.zookeeper zookeeper @@ -1983,6 +1994,16 @@ jackson-jaxrs-json-provider ${jackson.version} + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + com.fasterxml.jackson.core jackson-databind @@ -2086,6 +2107,12 @@ javax.servlet.jsp ${glassfish.jsp.version} + + + javax.servlet.jsp + javax.servlet.jsp-api + 2.3.1 + org.glassfish javax.el @@ -2550,10 +2577,6 @@ hadoop-hdfs ${hadoop-two.version} - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2595,10 +2618,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2643,10 +2662,6 @@ hadoop-common ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-beanutils commons-beanutils @@ -2697,10 +2712,6 @@ hadoop-minicluster ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient @@ -2819,10 +2830,6 @@ hadoop-hdfs ${hadoop-three.version} - - org.apache.htrace - htrace-core - com.sun.jersey jersey-core @@ -2868,10 +2875,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2952,10 +2955,6 @@ com.sun.jersey jersey-server - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -3020,10 +3019,6 @@ hadoop-minicluster ${hadoop-three.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient -- 2.16.1