From d617c5bdd4726f80969450f4fc004eda2dc7b97b Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Thu, 10 Aug 2017 16:16:20 -0500 Subject: [PATCH] HBASE-18577 WIP test well-formed-ness of our client jars. * Removes transitive hadoop deps from hbase-client that we don't use * removes curator-recipes dep from hbase-client that we don't use * removes jetty-util transitive from hbase-client that we don't use * moves junit from compile to test scope for hbase-client * adds an IT that checks the contents of jars * excludes our server side web adds from shaded jars * excludes proto source files from shaded jars * relocates codahale metrics, commons-crypto, and curator * finishes relocating jets3t * leaves as exposed dependencies jaxb-api, jersey, jaxb-impl, javax annotations, javax servlet jsp, javax inject, javax validation, javax servlet, javax ws rs, javassist, glassfish, jetty ( :( ) * moves logging backends to optional in shaded artifacts * updates maven dependency plugin --- hbase-client/pom.xml | 20 ++- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 177 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 66 ++++++++ hbase-shaded/pom.xml | 100 +++++++++++- pom.xml | 7 + 5 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 45c43e9..f9aa586 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -119,6 +119,13 @@ org.apache.hbase hbase-hadoop2-compat + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + org.apache.hbase @@ -193,13 +200,14 @@ metrics-core - org.mockito - mockito-all + junit + junit test - org.apache.curator - curator-recipes + org.mockito + mockito-all + test org.apache.curator @@ -273,6 +281,10 @@ jetty + org.mortbay.jetty + jetty-util + + com.sun.jersey jersey-server diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml new file mode 100644 index 0000000..e2a4af0 --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -0,0 +1,177 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-check-invariants + pom + + Enforces our invariants for our shaded artifacts. + Apache HBase Shaded Packaging Invariants + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-server + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-all + provided + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-3 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + check-jar-contents + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000..0ccdf7c --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allowed_expr="(^org/$|^org/apache/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop, which by +# convention should be in a path that looks like org/apache/hadoop +allowed_expr+="|^org/apache/hadoop/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" +# * Hadoop's and HBase's default configuration files, which have the form +# "_module_-default.xml" +allowed_expr+="|^[^-]*-default.xml$" +# * Hadoop's versioning properties files, which have the form +# "_module_-version-info.properties" +allowed_expr+="|^[^-]*-version-info.properties$" +# * Hadoop's application classloader properties file. +allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +# public suffix list used by httpcomponents +allowed_expr+="|^mozilla/$" +allowed_expr+="|^mozilla/public-suffix-list.txt$" +# Comes from commons-configuration, not sure if relocatable. +allowed_expr+="|^digesterRules.xml$" +allowed_expr+="|^properties.dtd$" +allowed_expr+="|^PropertyList-1.0.dtd$" + + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename ${artifact})'" + fi +done + +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 6bccc7d..1370570 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -40,6 +40,7 @@ hbase-shaded-client hbase-shaded-server + hbase-shaded-check-invariants @@ -48,6 +49,23 @@ ${project.version} true + + junit + junit + test + + + + log4j + log4j + true + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + true + @@ -120,7 +138,6 @@ false false true - ${project.build.directory}/dependency-reduced-pom.xml false @@ -131,11 +148,38 @@ org.apache.htrace:* log4j:* commons-logging:* + + javax.annotation:* + javax.xml.bind:jaxb-api + javax.inject:* + javax.validation:* + javax.servlet:* + javax.servlet.jsp:* + javax.ws.rs:* + com.sun.jersey:* + com.sun.jersey.contribs:* + com.sun.xml.bind:* + org.javassist:* + org.glassfish.web:* + org.glassfish:* + org.glassfish.jersey.containers:* + org.glassfish.jersey.core:* + org.glassfish.jersey.bundles.repackaged:* + org.glassfish.jersey.media:* + org.glassfish.hk2:* + org.glassfish.hk2.external:* + org.eclipse.jetty:* + org.eclipse.jetty.orbit:* + org.eclipse.jetty.toolchain:* + com.codahale + org.apache.hadoop.hbase.shaded.com.codahale + + com.google org.apache.hadoop.hbase.shaded.com.google @@ -215,6 +259,11 @@ org.jets3t org.apache.hadoop.hbase.shaded.org.jets3t + + + contribs.mx + org.apache.hadoop.hbase.shaded.contribs.mx + org.objectweb org.apache.hadoop.hbase.shaded.org.objectweb @@ -227,6 +276,10 @@ org.apache.hadoop.hbase.shaded.org.apache.avro + org.apache.curator + org.apache.hadoop.hbase.shaded.org.apache.curator + + org.apache.directory org.apache.hadoop.hbase.shaded.org.apache.directory @@ -269,6 +322,10 @@ org.apache.hadoop.hbase.shaded.org.apache.commons.configuration + org.apache.commons.crypto + org.apache.hadoop.hbase.shaded.org.apache.commons.crypto + + org.apache.commons.daemon org.apache.hadoop.hbase.shaded.org.apache.commons.daemon @@ -339,6 +396,9 @@ LICENSE.txt ASL2.0 + LICENSE-junit.txt + + overview.html @@ -356,6 +416,44 @@ META-INF/ECLIPSEF.RSA + + + org.apache.hbase:hbase-server + + hbase-webapps/* + hbase-webapps/**/* + + + + + org.apache.hadoop:hadoop-yarn-common + + webapps/* + webapps/**/* + + + + + *:* + + *.proto + **/*.proto + + + + + org.apache.commons:commons-math3 + + assets/org/apache/commons/math3/**/* + + + + + org.apache.hadoop:* + + mapred-default.xml.orig + + diff --git a/pom.xml b/pom.xml index 5040361..0e9abeb 100755 --- a/pom.xml +++ b/pom.xml @@ -558,6 +558,11 @@ org.apache.maven.plugins + maven-dependency-plugin + ${maven.dependency.version} + + + org.apache.maven.plugins maven-javadoc-plugin @@ -1432,6 +1437,7 @@ 3.3.0 2.17 3.6.1 + 3.0.1 2.10 2.5.2 3.0.2 @@ -1463,6 +1469,7 @@ hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar hbase-rsgroup-${project.version}-tests.jar + bash 2.19.1 surefire-junit47 -- 2.7.2