From 92663bac06081f277ab73764ad19a4fab50ea91f Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Thu, 10 Aug 2017 16:16:20 -0500 Subject: [PATCH] HBASE-18577 WIP test well-formed-ness of our client jars. * Removes transitive hadoop deps from hbase-client that we don't use * removes curator-recipes dep from hbase-client that we don't use * removes jetty-util transitive from hbase-client that we don't use * moves junit from compile to test scope for hbase-client * adds an IT that checks the contents of jars * excludes our server side web apps and related deps (jaxb, jersey, jetty) from shaded jars * excludes proto source files from shaded jars * relocates codahale metrics, commons-crypto, and curator * finishes relocating jets3t * moves logging backends to optional in shaded artifacts * updates maven dependency plugin --- hbase-client/pom.xml | 20 ++- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 177 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 66 ++++++++ hbase-shaded/hbase-shaded-server/pom.xml | 65 ++++++++ hbase-shaded/pom.xml | 77 ++++++++- pom.xml | 7 + 6 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 23278bf..67c03f3 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -119,6 +119,13 @@ org.apache.hbase hbase-hadoop2-compat + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + org.apache.hbase @@ -193,13 +200,14 @@ metrics-core - org.mockito - mockito-all + junit + junit test - org.apache.curator - curator-recipes + org.mockito + mockito-all + test org.apache.curator @@ -273,6 +281,10 @@ jetty + org.mortbay.jetty + jetty-util + + com.sun.jersey jersey-server diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml new file mode 100644 index 0000000..e2a4af0 --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -0,0 +1,177 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-check-invariants + pom + + Enforces our invariants for our shaded artifacts. + Apache HBase Shaded Packaging Invariants + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-server + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-all + provided + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-3 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + check-jar-contents + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000..502d093 --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allowed_expr="(^org/$|^org/apache/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop, which by +# convention should be in a path that looks like org/apache/hadoop +allowed_expr+="|^org/apache/hadoop/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" +# * Hadoop's and HBase's default configuration files, which have the form +# "_module_-default.xml" +allowed_expr+="|^[^-]*-default.xml$" +# * Hadoop's versioning properties files, which have the form +# "_module_-version-info.properties" +allowed_expr+="|^[^-]*-version-info.properties$" +# * Hadoop's application classloader properties file. +allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +# public suffix list used by httpcomponents +allowed_expr+="|^mozilla/$" +allowed_expr+="|^mozilla/public-suffix-list.txt$" +# Comes from commons-configuration, not sure if relocatable. +allowed_expr+="|^digesterRules.xml$" +allowed_expr+="|^properties.dtd$" +allowed_expr+="|^PropertyList-1.0.dtd$" + + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'" + fi +done + +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/hbase-shaded-server/pom.xml b/hbase-shaded/hbase-shaded-server/pom.xml index 86166cb..a1a8633 100644 --- a/hbase-shaded/hbase-shaded-server/pom.xml +++ b/hbase-shaded/hbase-shaded-server/pom.xml @@ -51,6 +51,71 @@ org.apache.hbase hbase-server + + + + javax.xml.bind + jaxb-api + + + + javax.ws.rs + javax.ws.rs-api + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-client + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-util-ajax + + + org.eclipse.jetty + jetty-jsp + + + org.eclipse.jetty + jetty-webapp + + + org.glassfish.jersey.containers + jersey-container-servlet-core + + + org.glassfish.jersey.media + jersey-media-json-jackson1 + + diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 25f3c59..f592409 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -40,6 +40,7 @@ hbase-shaded-client hbase-shaded-server + hbase-shaded-check-invariants @@ -48,6 +49,23 @@ ${project.version} true + + junit + junit + test + + + + log4j + log4j + true + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + true + @@ -120,7 +138,6 @@ false false true - ${project.build.directory}/dependency-reduced-pom.xml false @@ -136,6 +153,10 @@ + com.codahale + org.apache.hadoop.hbase.shaded.com.codahale + + com.google org.apache.hadoop.hbase.shaded.com.google @@ -215,6 +236,11 @@ org.jets3t org.apache.hadoop.hbase.shaded.org.jets3t + + + contribs.mx + org.apache.hadoop.hbase.shaded.contribs.mx + org.objectweb org.apache.hadoop.hbase.shaded.org.objectweb @@ -227,6 +253,10 @@ org.apache.hadoop.hbase.shaded.org.apache.avro + org.apache.curator + org.apache.hadoop.hbase.shaded.org.apache.curator + + org.apache.directory org.apache.hadoop.hbase.shaded.org.apache.directory @@ -269,6 +299,10 @@ org.apache.hadoop.hbase.shaded.org.apache.commons.configuration + org.apache.commons.crypto + org.apache.hadoop.hbase.shaded.org.apache.commons.crypto + + org.apache.commons.daemon org.apache.hadoop.hbase.shaded.org.apache.commons.daemon @@ -339,6 +373,9 @@ LICENSE.txt ASL2.0 + LICENSE-junit.txt + + overview.html @@ -356,6 +393,44 @@ META-INF/ECLIPSEF.RSA + + + org.apache.hbase:hbase-server + + hbase-webapps/* + hbase-webapps/**/* + + + + + org.apache.hadoop:hadoop-yarn-common + + webapps/* + webapps/**/* + + + + + *:* + + *.proto + **/*.proto + + + + + org.apache.commons:commons-math3 + + assets/org/apache/commons/math3/**/* + + + + + org.apache.hadoop:* + + mapred-default.xml.orig + + diff --git a/pom.xml b/pom.xml index 464d42f..a92db8e 100755 --- a/pom.xml +++ b/pom.xml @@ -568,6 +568,11 @@ org.apache.maven.plugins + maven-dependency-plugin + ${maven.dependency.version} + + + org.apache.maven.plugins maven-javadoc-plugin @@ -1443,6 +1448,7 @@ 3.3.0 2.17 3.6.1 + 3.0.1 2.10 2.5.2 3.0.2 @@ -1475,6 +1481,7 @@ hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar hbase-rsgroup-${project.version}-tests.jar + bash 2.19.1 surefire-junit47 -- 2.7.2