From 19623b5fdee737acec65fb76fdb99961ac8c39f6 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Thu, 10 Aug 2017 16:16:20 -0500 Subject: [PATCH] HBASE-18577 WIP test well-formed-ness of our client jars. * adds an IT that checks the contents of jars * excludes our server side web adds from shaded jars * excludes proto source files from shaded jars * updates maven dependency plugin --- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 177 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 61 +++++++ hbase-shaded/pom.xml | 18 ++- pom.xml | 7 + 4 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 hbase-shaded/hbase-shaded-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml new file mode 100644 index 0000000..e2a4af0 --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -0,0 +1,177 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-check-invariants + pom + + Enforces our invariants for our shaded artifacts. + Apache HBase Shaded Packaging Invariants + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-server + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-all + provided + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-3 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + check-jar-contents + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000..8847273 --- /dev/null +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allowed_expr="(^org/$|^org/apache/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop, which by +# convention should be in a path that looks like org/apache/hadoop +allowed_expr+="|^org/apache/hadoop/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" + +## * whatever under the "webapps" directory; for things shipped by yarn +#allowed_expr+="|^webapps/" +## * Hadoop's default configuration files, which have the form +## "_module_-default.xml" +#allowed_expr+="|^[^-]*-default.xml$" +## * Hadoop's versioning properties files, which have the form +## "_module_-version-info.properties" +#allowed_expr+="|^[^-]*-version-info.properties$" +## * Hadoop's application classloader properties file. +#allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename ${artifact})'" + fi +done + +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 6bccc7d..6c42158 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -40,6 +40,7 @@ hbase-shaded-client hbase-shaded-server + hbase-shaded-check-invariants @@ -120,7 +121,6 @@ false false true - ${project.build.directory}/dependency-reduced-pom.xml false @@ -356,6 +356,22 @@ META-INF/ECLIPSEF.RSA + + + org.apache.hbase:hbase-server + + hbase-webapps/* + hbase-webapps/**/* + + + + + *:* + + *.proto + **/*.proto + + diff --git a/pom.xml b/pom.xml index 7648e8e..708caa6 100644 --- a/pom.xml +++ b/pom.xml @@ -558,6 +558,11 @@ org.apache.maven.plugins + maven-dependency-plugin + ${maven.dependency.version} + + + org.apache.maven.plugins maven-javadoc-plugin @@ -1420,6 +1425,7 @@ 3.3.0 2.17 3.6.1 + 3.0.1 2.10 2.5.2 3.0.2 @@ -1451,6 +1457,7 @@ hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar hbase-rsgroup-${project.version}-tests.jar + bash 2.19.1 surefire-junit47 -- 2.7.2