From 3cc346d7564eee7bd368c8cb983db5baf6f3d6a5 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 9 Apr 2018 13:37:44 -0500 Subject: [PATCH 1/5] HBASE-20332 shaded mapreduce module shouldn't include hadoop * modify the jar checking script to take args; make hadoop stuff optional * separate out checking the artifacts that have hadoop vs those that don't. * * Unfortunately means we need two modules for checking things * * put in a safety check that the support script for checking jar contents is maintained in both modules * * have to carve out an exception for o.a.hadoop.metrics2. :( * fix duplicated class warning * clean up dependencies in hbase-server and some modules that depend on it. * allow Hadoop to have its own htrace where it needs it * add a precommit check to make sure we're not using old htrace imports --- hbase-backup/pom.xml | 12 +- .../src/main/resources/hbase/checkstyle.xml | 4 +- hbase-client/pom.xml | 4 - hbase-common/pom.xml | 12 -- hbase-endpoint/pom.xml | 14 -- hbase-examples/pom.xml | 12 -- hbase-external-blockcache/pom.xml | 4 - hbase-hadoop2-compat/pom.xml | 6 - hbase-it/pom.xml | 6 - hbase-mapreduce/pom.xml | 30 +-- hbase-replication/pom.xml | 4 - hbase-rest/pom.xml | 19 +- hbase-rsgroup/pom.xml | 4 - hbase-server/pom.xml | 103 +++++----- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 54 ++++-- .../resources/ensure-jars-have-correct-contents.sh | 92 +++++++-- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++++++++++++++++- .../pom.xml | 215 +++++++++++++++++++++ .../resources/ensure-jars-have-correct-contents.sh | 129 +++++++++++++ hbase-shaded/pom.xml | 13 ++ hbase-shell/pom.xml | 14 -- hbase-testing-util/pom.xml | 16 -- hbase-thrift/pom.xml | 16 -- pom.xml | 65 +++---- 24 files changed, 773 insertions(+), 265 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml create mode 100644 hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 7afd51eaf0..00a996f51e 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -154,10 +154,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -264,9 +260,6 @@ 3.0 - - 3.0-SNAPSHOT - org.apache.hadoop @@ -276,6 +269,11 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml index 7ad797cf1f..148e2564e2 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml @@ -86,8 +86,10 @@ org.apache.commons.lang, org.apache.curator.shaded, org.apache.hadoop.classification, + org.apache.htrace, org.apache.htrace.shaded, - org.codehaus.jackson"/> + org.codehaus.jackson, + org.htrace"/> diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index f6247e30e3..bb99eec4ea 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -234,10 +234,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 5ae8e0b637..0aaccb89e3 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -314,12 +314,6 @@ hadoop-common - - - org.apache.htrace - htrace-core - - @@ -366,12 +360,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml index e9a8cf7132..a831d3a398 100644 --- a/hbase-endpoint/pom.xml +++ b/hbase-endpoint/pom.xml @@ -260,12 +260,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -296,10 +290,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -343,10 +333,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index c74c1ba24e..e417b2d212 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -232,12 +232,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - @@ -287,12 +281,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index 24c33ebe48..2479b46039 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -224,10 +224,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index ab33c72eb8..c1e42129da 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -167,12 +167,6 @@ limitations under the License. org.apache.hadoop hadoop-common ${hadoop.version} - - - org.apache.htrace - htrace-core - - org.apache.commons diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 9b1a167155..4c17245a9e 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -335,12 +335,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index af80737644..2bf693bc3d 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -196,6 +196,15 @@ org.apache.hbase hbase-server + + + + commons-logging + commons-logging + + org.apache.hbase @@ -246,10 +255,19 @@ junit test + com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + @@ -284,10 +302,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t @@ -334,10 +348,6 @@ org.apache.hadoop hadoop-hdfs - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -377,10 +387,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - org.apache.zookeeper zookeeper diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index d05c60ef64..b999c1d6cc 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -155,10 +155,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 617f254a2f..d06feec283 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -299,6 +299,19 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + + org.codehaus.jettison + jettison + + + stax + stax-api + + + org.glassfish.web @@ -374,12 +387,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml index 2d9a10d16c..1cc38549b9 100644 --- a/hbase-rsgroup/pom.xml +++ b/hbase-rsgroup/pom.xml @@ -198,10 +198,6 @@ org.apache.hadoop hadoop-common - - org.apache.htrace - htrace-core - net.java.dev.jets3t jets3t diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 11361d886b..e9daf93aae 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -71,6 +71,14 @@ org.apache.maven.plugins maven-remote-resources-plugin 1.5 + + + + org.apache.hbase + hbase-resource-bundle + ${project.version} + + default @@ -392,12 +400,6 @@ org.apache.hbase hbase-metrics - - - org.apache.hbase - hbase-resource-bundle - true - commons-codec commons-codec @@ -437,19 +439,16 @@ jetty-webapp - + org.glassfish.web javax.servlet.jsp + - org.codehaus.jettison - jettison - - - stax - stax-api - - + javax.servlet.jsp + javax.servlet.jsp-api @@ -501,9 +500,20 @@ javax.servlet javax.servlet-api + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + - javax.ws.rs - javax.ws.rs-api + com.fasterxml.jackson.core + jackson-annotations @@ -511,11 +521,6 @@ org.apache.htrace htrace-core4 - - org.apache.htrace - htrace-core - ${htrace-hadoop.version} - com.lmax disruptor @@ -556,6 +561,15 @@ httpcore test + + + commons-logging + commons-logging + compile + org.apache.commons commons-crypto @@ -674,34 +688,10 @@ - - org.apache.hadoop - hadoop-distcp - ${hadoop-two.version} - org.apache.hadoop hadoop-common - - org.apache.hadoop - hadoop-auth - - - org.apache.hadoop - hadoop-annotations - ${hadoop-two.version} - - - org.apache.hadoop - hadoop-client - - - com.google.guava - guava - - - org.apache.hadoop hadoop-mapreduce-client-core @@ -796,21 +786,32 @@ org.apache.hadoop - hadoop-distcp - ${hadoop-three.version} + hadoop-common org.apache.hadoop - hadoop-common + hadoop-hdfs + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core org.apache.hadoop hadoop-hdfs + test-jar + test org.apache.hadoop - hadoop-annotations - ${hadoop-three.version} + hadoop-mapreduce-client-jobclient + test-jar + test org.apache.hadoop diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7322769f0b..7ba4a41782 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -26,7 +26,7 @@ Enforces our invariants for our shaded artifacts. e.g. shaded clients have a specific set of transitive dependencies and shaded clients only contain classes that are in particular packages. Does the enforcement through - the maven-enforcer-plugin and and integration test. + the maven-enforcer-plugin and integration test. Apache HBase Shaded Packaging Invariants @@ -34,11 +34,15 @@ - - org.apache.hbase - hbase-shaded-client - ${project.version} - + org.apache.hbase hbase-shaded-mapreduce @@ -113,6 +117,8 @@ com.github.stephenc.findbugs:* org.apache.htrace:* + + org.apache.hadoop:* @@ -158,18 +164,37 @@ - org.codehaus.mojo exec-maven-plugin 1.6.0 + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + check-jar-contents integration-test @@ -180,6 +205,9 @@ ${shell-executable} ${project.build.testOutputDirectory} false + ensure-jars-have-correct-contents.sh ${hbase-client-artifacts} diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 8bda8ce953..eff1d20302 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -15,33 +15,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar] -# -# accepts a single command line argument with a colon separated list of -# paths to jars to check. Iterates through each such passed jar and checks -# all the contained paths to make sure they follow the below constructed -# safe list. +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") -# we have to allow the directories that lead to the org/apache/hadoop dir -allowed_expr="(^org/$|^org/apache/$" +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" # We allow the following things to exist in our client artifacts: -# * classes in packages that start with org.apache.hadoop, which by -# convention should be in a path that looks like org/apache/hadoop -allowed_expr+="|^org/apache/hadoop/" +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" # * classes in packages that start with org.apache.hbase allowed_expr+="|^org/apache/hbase/" # * whatever in the "META-INF" directory allowed_expr+="|^META-INF/" # * the folding tables from jcodings allowed_expr+="|^tables/" -# * Hadoop's and HBase's default configuration files, which have the form +# * HBase's default configuration files, which have the form # "_module_-default.xml" -allowed_expr+="|^[^-]*-default.xml$" -# * Hadoop's versioning properties files, which have the form -# "_module_-version-info.properties" -allowed_expr+="|^[^-]*-version-info.properties$" -# * Hadoop's application classloader properties file. -allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +allowed_expr+="|^hbase-default.xml$" # public suffix list used by httpcomponents allowed_expr+="|^mozilla/$" allowed_expr+="|^mozilla/public-suffix-list.txt$" @@ -51,12 +85,30 @@ allowed_expr+="|^properties.dtd$" allowed_expr+="|^PropertyList-1.0.dtd$" +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + allowed_expr+=")" declare -i bad_artifacts=0 declare -a bad_contents -IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") for artifact in "${artifact_list[@]}"; do - bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}")) + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) if [ ${#bad_contents[@]} -gt 0 ]; then echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" echo " Please check the following and either correct the build or update" diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml index cfcc357877..edc80cd151 100644 --- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml +++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml @@ -62,6 +62,10 @@ + org.apache.hbase hbase-mapreduce @@ -137,10 +141,6 @@ org.eclipse.jetty jetty-webapp - - org.glassfish.web - javax.servlet.jsp - org.glassfish.jersey.core jersey-server @@ -149,6 +149,17 @@ org.glassfish.jersey.containers jersey-container-servlet-core + + + org.glassfish.web + javax.servlet.jsp + + + javax.servlet.jsp + javax.servlet.jsp-api + @@ -158,12 +169,175 @@ release - - org.apache.maven.plugins - maven-shade-plugin - + + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + org.apache.hadoop:* + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + + + + + hadoop-2.0 + + + + !hadoop.profile + + + + + org.apache.hadoop + hadoop-common + provided + + + net.java.dev.jets3t + jets3t + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + javax.servlet + servlet-api + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + com.google.code.findbugs + jsr305 + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + javax.servlet.jsp + jsp-api + + + javax.servlet + servlet-api + + + io.netty + netty + + + stax + stax-api + + + xerces + xercesImpl + + + ${hadoop-two.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + com.google.guava + guava + + + + + org.apache.hadoop + hadoop-auth + provided + + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + ${hadoop-three.version} + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-auth + provided + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml new file mode 100644 index 0000000000..07789f4712 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml @@ -0,0 +1,215 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 3.0.0-SNAPSHOT + ../.. + + hbase-shaded-with-hadoop-check-invariants + pom + + + Enforces our invariants for our shaded artifacts. e.g. shaded clients have + a specific set of transitive dependencies and shaded clients only contain + classes that are in particular packages. Does the enforcement through + the maven-enforcer-plugin and integration test. + + Apache HBase Shaded Packaging Invariants (with Hadoop bundled) + + + + + + + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + + com.github.stephenc.findbugs + findbugs-annotations + provided + + + log4j + log4j + provided + + + + junit + junit + provided + + + org.mockito + mockito-core + provided + + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-6 + + + + + enforce-banned-dependencies + + enforce + + + true + + + + + + org.slf4j:* + log4j:* + commons-logging:* + + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + + org.apache.htrace:* + + + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + test-resources + pre-integration-test + + testResources + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + put-client-artifacts-in-a-property + pre-integration-test + + build-classpath + + + provided + true + hbase-client-artifacts + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + + make-sure-validation-files-are-in-sync + validate + + exec + + + diff + false + + ../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + ../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh + + + + + + check-jar-contents-for-stuff-with-hadoop + integration-test + + exec + + + ${shell-executable} + ${project.build.testOutputDirectory} + false + + ensure-jars-have-correct-contents.sh + --allow-hadoop + ${hbase-client-artifacts} + + + + + + + + + diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh new file mode 100644 index 0000000000..eff1d20302 --- /dev/null +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +function usage { + echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]" + echo "" + echo " accepts a single command line argument with a colon separated list of" + echo " paths to jars to check. Iterates through each such passed jar and checks" + echo " all the contained paths to make sure they follow the below constructed" + echo " safe list." + echo "" + echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list" + echo " of allowed jar contents. default: false" + echo " --debug print more info to stderr" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare allow_hadoop +declare debug +while [ $# -gt 0 ] +do + case "$1" in + --allow-hadoop) shift; allow_hadoop="true";; + --debug) shift; debug="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have jars to check. +if [ $# -lt 1 ]; then + usage +fi +if [ -n "${debug}" ]; then + echo "[DEBUG] Checking on jars: $*" >&2 + echo "jar command is: $(which jar)" >&2 + echo "grep command is: $(which grep)" >&2 + grep -V >&2 || true +fi + +IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1") + +# we have to allow the directories that lead to the hbase dirs +allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$" +# We allow the following things to exist in our client artifacts: +# * classes in packages that start with org.apache.hadoop.hbase, which by +# convention should be in a path that looks like org/apache/hadoop/hbase +allowed_expr+="|^org/apache/hadoop/hbase" +# * classes in packages that start with org.apache.hbase +allowed_expr+="|^org/apache/hbase/" +# * whatever in the "META-INF" directory +allowed_expr+="|^META-INF/" +# * the folding tables from jcodings +allowed_expr+="|^tables/" +# * HBase's default configuration files, which have the form +# "_module_-default.xml" +allowed_expr+="|^hbase-default.xml$" +# public suffix list used by httpcomponents +allowed_expr+="|^mozilla/$" +allowed_expr+="|^mozilla/public-suffix-list.txt$" +# Comes from commons-configuration, not sure if relocatable. +allowed_expr+="|^digesterRules.xml$" +allowed_expr+="|^properties.dtd$" +allowed_expr+="|^PropertyList-1.0.dtd$" + + +if [ -n "${allow_hadoop}" ]; then + # * classes in packages that start with org.apache.hadoop, which by + # convention should be in a path that looks like org/apache/hadoop + allowed_expr+="|^org/apache/hadoop/" + # * Hadoop's default configuration files, which have the form + # "_module_-default.xml" + allowed_expr+="|^[^-]*-default.xml$" + # * Hadoop's versioning properties files, which have the form + # "_module_-version-info.properties" + allowed_expr+="|^[^-]*-version-info.properties$" + # * Hadoop's application classloader properties file. + allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" +else + # We have some classes for integrating with the Hadoop Metrics2 system + # that have to be in a particular package space due to access rules. + allowed_expr+="|^org/apache/hadoop/metrics2" +fi + + +allowed_expr+=")" +declare -i bad_artifacts=0 +declare -a bad_contents +for artifact in "${artifact_list[@]}"; do + bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true)) + if [ ${#bad_contents[@]} -gt 0 ]; then + echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" + echo " Please check the following and either correct the build or update" + echo " the allowed list with reasoning." + echo "" + for bad_line in "${bad_contents[@]}"; do + echo " ${bad_line}" + done + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'" + fi +done + +# if there was atleast one bad artifact, exit with failure +if [ "${bad_artifacts}" -gt 0 ]; then + exit 1 +fi diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 24c515844e..93b122fe08 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -42,6 +42,7 @@ hbase-shaded-client hbase-shaded-mapreduce hbase-shaded-check-invariants + hbase-shaded-with-hadoop-check-invariants @@ -118,6 +119,7 @@ maven-shade-plugin + aggregate-into-a-jar-with-relocated-third-parties package shade @@ -449,12 +451,23 @@ META-INF/ECLIPSEF.RSA + + + commons-beanutils:commons-beanutils-core + + org/apache/commons/collections/*.class + + org.apache.hbase:hbase-server hbase-webapps/* hbase-webapps/**/* + **/*_jsp.class diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 8eaefaab83..85f0415605 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -277,12 +277,6 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -341,10 +335,6 @@ hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -408,10 +398,6 @@ org.apache.hadoop hadoop-minicluster - - org.apache.htrace - htrace-core - com.google.guava guava diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml index 0f1b86cf83..032de61e3d 100644 --- a/hbase-testing-util/pom.xml +++ b/hbase-testing-util/pom.xml @@ -140,12 +140,6 @@ org.apache.hadoop hadoop-common compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop @@ -201,10 +195,6 @@ hadoop-minicluster compile - - org.apache.htrace - htrace-core - com.google.guava guava @@ -242,12 +232,6 @@ org.apache.hadoop hadoop-minicluster compile - - - org.apache.htrace - htrace-core - - org.apache.hadoop diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index 0142ccdb59..aec3cb4668 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -498,22 +498,12 @@ org.apache.hadoop hadoop-common - - - org.apache.htrace - htrace-core - - org.apache.hadoop hadoop-minicluster test - - org.apache.htrace - htrace-core - com.google.guava guava @@ -571,12 +561,6 @@ org.apache.hadoop hadoop-minicluster - - - org.apache.htrace - htrace-core - - diff --git a/pom.xml b/pom.xml index ed7a1722ba..a943125475 100755 --- a/pom.xml +++ b/pom.xml @@ -1448,8 +1448,11 @@ 11.0.2 hbase-hadoop2-compat src/main/assembly/hadoop-two-compat.xml - 0.5.0 + + 3.6.2.Final + 0.5.0 1.7.7 1.10 @@ -1477,7 +1480,6 @@ 4.12 1.3 4.2.0-incubating - 3.2.0-incubating 1.2.17 2.1.0 @@ -1605,7 +1607,8 @@ org.mortbay.jetty:servlet-api, javax.servlet:servlet-api: These are excluded because they are the same implementations. I chose org.mortbay.jetty:servlet-api-2.5 instead, which is a third implementation of the same, because Hadoop also uses this version - javax.servlet:jsp-api in favour of org.mortbay.jetty:jsp-api-2.1 + javax.servlet:jsp-api in favour of javax.servlet.jsp:javax.servlet.jsp-api:2.3.1 since it + is what glassfish's jspC jar uses and that's where we get our own need for a jsp-api. --> @@ -1920,6 +1923,14 @@ commons-math3 ${commons-math.version} + + + commons-logging + commons-logging + 1.2 + org.apache.zookeeper zookeeper @@ -1983,6 +1994,16 @@ jackson-jaxrs-json-provider ${jackson.version} + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + com.fasterxml.jackson.core jackson-databind @@ -2086,6 +2107,12 @@ javax.servlet.jsp ${glassfish.jsp.version} + + + javax.servlet.jsp + javax.servlet.jsp-api + 2.3.1 + org.glassfish javax.el @@ -2550,10 +2577,6 @@ hadoop-hdfs ${hadoop-two.version} - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2595,10 +2618,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2643,10 +2662,6 @@ hadoop-common ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-beanutils commons-beanutils @@ -2697,10 +2712,6 @@ hadoop-minicluster ${hadoop-two.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient @@ -2819,10 +2830,6 @@ hadoop-hdfs ${hadoop-three.version} - - org.apache.htrace - htrace-core - com.sun.jersey jersey-core @@ -2868,10 +2875,6 @@ test-jar test - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -2952,10 +2955,6 @@ com.sun.jersey jersey-server - - org.apache.htrace - htrace-core - javax.servlet.jsp jsp-api @@ -3020,10 +3019,6 @@ hadoop-minicluster ${hadoop-three.version} - - org.apache.htrace - htrace-core - commons-httpclient commons-httpclient -- 2.16.1 From 1e09eb4cecc2940588f97a8a790b534e06269329 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 24 Apr 2018 14:51:12 -0500 Subject: [PATCH 2/5] HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs. --- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 5 ++ .../hbase-shaded-client-byo-hadoop/pom.xml | 70 ++++++++++++++++++++++ hbase-shaded/hbase-shaded-client/pom.xml | 35 +++++++++-- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++-------- hbase-shaded/pom.xml | 6 ++ 5 files changed, 115 insertions(+), 31 deletions(-) create mode 100644 hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7ba4a41782..287a986568 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -48,6 +48,11 @@ hbase-shaded-mapreduce ${project.version} + + org.apache.hbase + hbase-shaded-client-byo-hadoop + ${project.version} + com.github.stephenc.findbugs diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml new file mode 100644 index 0000000000..c51a1af745 --- /dev/null +++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + hbase-shaded + org.apache.hbase + 3.0.0-SNAPSHOT + .. + + hbase-shaded-client-byo-hadoop + Apache HBase - Shaded - Client + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + maven-assembly-plugin + + true + + + + + + + org.apache.hbase + hbase-client + + + + + + release + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + + diff --git a/hbase-shaded/hbase-shaded-client/pom.xml b/hbase-shaded/hbase-shaded-client/pom.xml index 72a5b6058e..5ac3ef5adb 100644 --- a/hbase-shaded/hbase-shaded-client/pom.xml +++ b/hbase-shaded/hbase-shaded-client/pom.xml @@ -28,7 +28,7 @@ .. hbase-shaded-client - Apache HBase - Shaded - Client + Apache HBase - Shaded - Client (with Hadoop bundled) @@ -51,6 +51,7 @@ org.apache.hbase hbase-client + ${project.version} @@ -59,10 +60,34 @@ release - - org.apache.maven.plugins - maven-shade-plugin - + + org.apache.maven.plugins + maven-shade-plugin + + + aggregate-into-a-jar-with-relocated-third-parties + + + + + + org.apache.hbase:hbase-resource-bundle + org.slf4j:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + org.apache.htrace:* + org.apache.yetus:* + log4j:* + commons-logging:* + + + + + + diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml index edc80cd151..598f3af938 100644 --- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml +++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml @@ -169,32 +169,10 @@ release - - - org.apache.maven.plugins - maven-shade-plugin - - - aggregate-into-a-jar-with-relocated-third-parties - - - - org.apache.hadoop:* - - org.apache.hbase:hbase-resource-bundle - org.slf4j:* - com.google.code.findbugs:* - com.github.stephenc.findbugs:* - org.apache.htrace:* - org.apache.yetus:* - log4j:* - commons-logging:* - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 93b122fe08..9eb30e0941 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -39,6 +39,7 @@ org.apache.hadoop.hbase.shaded + hbase-shaded-client-byo-hadoop hbase-shaded-client hbase-shaded-mapreduce hbase-shaded-check-invariants @@ -131,6 +132,11 @@ false + + org.apache.hadoop:* + org.apache.hbase:hbase-resource-bundle org.slf4j:* com.google.code.findbugs:* -- 2.16.1 From 9eca9005e121d2105cdc6159921bc7eb9b0eecee Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Wed, 7 Feb 2018 18:37:39 -0500 Subject: [PATCH 3/5] HBASE-19735 Create a client-tarball assembly Provides an extra client descriptor to build a second tarball with a reduced set of dependencies. Not of great impact now, but will build the way for better in the future. Signed-off-by: Sean Busbey Conflicts: hbase-assembly/pom.xml --- hbase-assembly/pom.xml | 33 +++-- .../src/main/assembly/client-components.xml | 92 ++++++++++++++ hbase-assembly/src/main/assembly/client.xml | 137 +++++++++++++++++++++ hbase-procedure/pom.xml | 4 +- hbase-spark/pom.xml | 6 + pom.xml | 10 ++ 6 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 hbase-assembly/src/main/assembly/client-components.xml create mode 100644 hbase-assembly/src/main/assembly/client.xml diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 5da105b24e..4fa859ac99 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -95,6 +95,7 @@ gnu ${assembly.file} + src/main/assembly/client.xml @@ -220,22 +221,22 @@ test - org.apache.hbase - hbase-hadoop-compat - - - com.google.guava - guava - - + org.apache.hbase + hbase-hadoop-compat + + + com.google.guava + guava + + - org.apache.hbase - ${compat.module} + org.apache.hbase + ${compat.module} - org.apache.hbase - hbase-shell + org.apache.hbase + hbase-shell org.apache.hbase @@ -315,6 +316,14 @@ jline jline + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml new file mode 100644 index 0000000000..2fd1b579e2 --- /dev/null +++ b/hbase-assembly/src/main/assembly/client-components.xml @@ -0,0 +1,92 @@ + + + + + + + + + ${project.basedir}/../target/site + docs + + + + ${project.basedir}/.. + . + + CHANGES.txt + README.txt + + 0644 + + + + ${project.basedir}/../conf + conf + 0644 + 0755 + + + + + ${project.basedir}/../bin + bin + + get-active-master.rb + hbase + hbase-common.sh + hbase-config.sh + hbase-jruby + hirb.rb + + + 0755 + 0755 + + + + ${project.basedir}/../bin + bin + + hbase.cmd + hbase-config.cmd + + + + + ${project.basedir}/../hbase-shell/src/main/ruby + lib/ruby + 0644 + 0755 + + + + ${project.basedir}/../hbase-server/target/native + lib/native + 0755 + 0755 + + *.so + *.dylib + + + + diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml new file mode 100644 index 0000000000..79519613c3 --- /dev/null +++ b/hbase-assembly/src/main/assembly/client.xml @@ -0,0 +1,137 @@ + + + + + + client-bin + + tar.gz + + + hbase-${project.version}-client + + src/main/assembly/client-components.xml + + + + true + + + org.apache.hbase:hbase-annotations + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-hadoop2-compat + org.apache.hbase:hbase-mapreduce + org.apache.hbase:hbase-metrics + org.apache.hbase:hbase-metrics-api + org.apache.hbase:hbase-procedure + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-server + org.apache.hbase:hbase-zookeeper + + + + lib + false + + + + + com.sun.jersey:* + com.sun.jersey.contribs:* + + org.apache.hbase:hbase-external-blockcache + org.apache.hbase:hbase-http + org.apache.hbase:hbase-replication + org.apache.hbase:hbase-rest + org.apache.hbase:hbase-rsgroup + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + org.apache.hbase:hbase-shell + org.apache.hbase:hbase-thrift + org.jruby:jruby-complete + + + + + + + true + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + + + shaded-lib + false + + + + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + + false + + + + + + + + + ${project.build.directory}/maven-shared-archive-resources/META-INF/LICENSE + . + LICENSE.txt + unix + + + ${project.build.directory}/NOTICE.aggregate + . + NOTICE.txt + unix + + + ${basedir}/src/main/resources/META-INF/LEGAL + . + LEGAL + unix + + + + + + + lib/ruby + + org.jruby:jruby-complete + + + + + diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index 3fedda8e19..b89acdb8a6 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -57,8 +57,8 @@ org.apache.hbase hbase-common - ${project.version} - tests + test-jar + test org.apache.hbase diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml index 7654be49bd..a5f96b4b46 100644 --- a/hbase-spark/pom.xml +++ b/hbase-spark/pom.xml @@ -236,6 +236,12 @@ hbase-common ${project.version} + + org.apache.hbase + hbase-common + test-jar + test + org.apache.hbase hbase-annotations diff --git a/pom.xml b/pom.xml index a943125475..b9a1b7725e 100755 --- a/pom.xml +++ b/pom.xml @@ -1853,6 +1853,16 @@ test-jar test + + org.apache.hbase + hbase-shaded-client + ${project.version} + + + org.apache.hbase + hbase-shaded-mapreduce + ${project.version} + com.github.stephenc.findbugs -- 2.16.1 From a7706ea09659fb74702f961def130e682c317c68 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Fri, 18 May 2018 11:11:42 -0500 Subject: [PATCH 4/5] HBASE-20615 emphasize shaded artifacts in client tarball. --- bin/hbase | 226 ++++++++++++++++++--- bin/hbase-config.sh | 10 + hbase-assembly/pom.xml | 21 +- .../src/main/assembly/client-components.xml | 43 +++- hbase-assembly/src/main/assembly/client.xml | 131 ++++++------ hbase-assembly/src/main/assembly/components.xml | 3 +- .../src/main/assembly/hadoop-two-compat.xml | 80 +++++++- pom.xml | 6 + 8 files changed, 417 insertions(+), 103 deletions(-) diff --git a/bin/hbase b/bin/hbase index 4f1c854dfa..559a02e9f8 100755 --- a/bin/hbase +++ b/bin/hbase @@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then in_dev_env=true fi +# Detect if we are in the omnibus tarball +in_omnibus_tarball="false" +if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then + in_omnibus_tarball="true" +fi + read -d '' options_string << EOF Options: - --config DIR Configuration direction to use. Default: ./conf - --hosts HOSTS Override the list in 'regionservers' file - --auth-as-server Authenticate to ZooKeeper using servers configuration + --config DIR Configuration direction to use. Default: ./conf + --hosts HOSTS Override the list in 'regionservers' file + --auth-as-server Authenticate to ZooKeeper using servers configuration + --internal-classpath Skip attempting to use client facing jars (WARNING: unstable results between versions) EOF # if no args specified, show usage if [ $# = 0 ]; then @@ -87,16 +94,18 @@ if [ $# = 0 ]; then echo " shell Run the HBase shell" echo " hbck Run the hbase 'fsck' tool" echo " snapshot Tool for managing snapshots" - echo " wal Write-ahead-log analyzer" - echo " hfile Store file analyzer" - echo " zkcli Run the ZooKeeper shell" - echo " master Run an HBase HMaster node" - echo " regionserver Run an HBase HRegionServer node" - echo " zookeeper Run a ZooKeeper server" - echo " rest Run an HBase REST server" - echo " thrift Run the HBase Thrift server" - echo " thrift2 Run the HBase Thrift2 server" - echo " clean Run the HBase clean up script" + if [ "${in_omnibus_tarball}" = "true" ]; then + echo " wal Write-ahead-log analyzer" + echo " hfile Store file analyzer" + echo " zkcli Run the ZooKeeper shell" + echo " master Run an HBase HMaster node" + echo " regionserver Run an HBase HRegionServer node" + echo " zookeeper Run a ZooKeeper server" + echo " rest Run an HBase REST server" + echo " thrift Run the HBase Thrift server" + echo " thrift2 Run the HBase Thrift2 server" + echo " clean Run the HBase clean up script" + fi echo " classpath Dump hbase CLASSPATH" echo " mapredcp Dump CLASSPATH entries required by mapreduce" echo " pe Run PerformanceEvaluation" @@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do fi done +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +# Allow this functionality to be disabled +if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then + HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) +fi + # Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; +declare shaded_jar + +if [ "${INTERNAL_CLASSPATH}" != "true" ]; then + # find our shaded jars + declare shaded_client + declare shaded_client_byo_hadoop + declare shaded_mapreduce + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do + if [[ "${f}" =~ byo-hadoop ]]; then + shaded_client_byo_hadoop="${f}" + else + shaded_client="${f}" + fi + done + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do + shaded_mapreduce="${f}" + done + + # If command can use our shaded client, use it + declare -a commands_in_client_jar=("classpath" "version") + for c in "${commands_in_client_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client_byo_hadoop}" + else + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client}" + fi + break + fi + done + + # If command needs our shaded mapreduce, use it + # N.B "mapredcp" is not included here because in the shaded case it skips our built classpath + declare -a commands_in_mr_jar=("hbck" "snapshot" "canary" "regionsplitter" "pre-upgrade") + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + + # Some commands specifically only can use shaded mapreduce when we'll get a full hadoop classpath at runtime + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + declare -a commands_in_mr_need_hadoop=("backup" "restore" "rowcounter" "cellcounter") + for c in "${commands_in_mr_need_hadoop[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + fi +fi + + +if [ -n "${shaded_jar}" ] && [ -f "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_jar}" +# fall through to grabbing all the lib jars and hope we're in the omnibus tarball +# +# N.B. shell specifically can't rely on the shaded artifacts because RSGroups is only +# available as non-shaded +# +# N.B. pe and ltt can't easily rely on shaded artifacts because they live in hbase-mapreduce:test-jar +# and need some other jars that haven't been relocated. Currently enumerating that list +# is too hard to be worth it. +# +else + for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + # make it easier to check for shaded/not later on. + shaded_jar="" +fi +for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + CLASSPATH="${CLASSPATH}:${f}" + fi done # default log directory & file @@ -201,9 +300,9 @@ fi function append_path() { if [ -z "$1" ]; then - echo $2 + echo "$2" else - echo $1:$2 + echo "$1:$2" fi } @@ -214,18 +313,34 @@ if [ "$HBASE_LIBRARY_PATH" != "" ]; then JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") fi -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -# Allow this functionality to be disabled -if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then - HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) - if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +#If configured and available, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" "${HADOOP_IN_PATH}" \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") fi + CLASSPATH=$(append_path "${CLASSPATH}" "$(${HADOOP_IN_PATH} classpath 2>/dev/null)") +else + # Otherwise, if we're providing Hadoop we should include htrace 3 if we were built with a version that needs it. + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core-3*.jar "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + # Some commands require special handling when using shaded jars. For these cases, we rely on hbase-shaded-mapreduce + # instead of hbase-shaded-client* because we make use of some IA.Private classes that aren't in the latter. However, + # we don't invoke them using the "hadoop jar" command so we need to ensure there are some Hadoop classes available + # when we're not doing runtime hadoop classpath lookup. + # + # luckily the set of classes we need are those packaged in the shaded-client. + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ] && [ -n "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_client:?We couldn\'t find the shaded client jar even though we did find the shaded MR jar. for command ${COMMAND} we need both. please use --internal-classpath as a workaround.}" + break + fi + done fi # Add user-specified CLASSPATH last @@ -250,11 +365,11 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` fi if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib") fi if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/lib/native/${JAVA_PLATFORM}") fi fi @@ -379,8 +494,30 @@ elif [ "$COMMAND" = "zkcli" ] ; then done elif [ "$COMMAND" = "backup" ] ; then CLASS='org.apache.hadoop.hbase.backup.BackupDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "restore" ] ; then CLASS='org.apache.hadoop.hbase.backup.RestoreDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + for f in "${HBASE_HOME}"/lib/commons-lang3*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "upgrade" ] ; then echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0." echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading." @@ -451,9 +588,24 @@ elif [ "$COMMAND" = "clean" ] ; then "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ exit $? elif [ "$COMMAND" = "mapredcp" ] ; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + if [ "${INTERNAL_CLASSPATH}" != "true" ] && [ -f "${shaded_jar}" ]; then + echo -n "${shaded_jar}" + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + echo -n ":${f}" + fi + done + echo "" + exit 0 + fi CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH + echo "$CLASSPATH" exit 0 elif [ "$COMMAND" = "pe" ] ; then CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' @@ -500,8 +652,20 @@ else fi HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" +# by now if we're running a command it means we need logging +for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi +done + # Exec unless HBASE_NOEXEC is set. export CLASSPATH +if [ "${DEBUG}" = "true" ]; then + echo "classpath=${CLASSPATH}" >&2 + HBASE_OPTS="${HBASE_OPTS} -Xdiag" +fi if [ "${HBASE_NOEXEC}" != "" ]; then "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh index 2e95ae71ad..1054751e63 100644 --- a/bin/hbase-config.sh +++ b/bin/hbase-config.sh @@ -84,6 +84,16 @@ do exit 1 fi shift + elif [ "--internal-classpath" = "$1" ] + then + shift + # shellcheck disable=SC2034 + INTERNAL_CLASSPATH="true" + elif [ "--debug" = "$1" ] + then + shift + # shellcheck disable=SC2034 + DEBUG="true" else # Presume we are at end of options and break break diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 4fa859ac99..fd1415fd20 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -190,6 +190,19 @@ + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-client-byo-hadoop + + + org.apache.hbase + hbase-shaded-mapreduce + org.apache.hbase @@ -316,14 +329,6 @@ jline jline - - org.apache.hbase - hbase-shaded-client - - - org.apache.hbase - hbase-shaded-mapreduce - diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml index 2fd1b579e2..2369f28d20 100644 --- a/hbase-assembly/src/main/assembly/client-components.xml +++ b/hbase-assembly/src/main/assembly/client-components.xml @@ -32,7 +32,8 @@ ${project.basedir}/.. . - CHANGES.txt + CHANGES.md + RELEASENOTES.md README.txt 0644 @@ -56,7 +57,6 @@ hbase-config.sh hbase-jruby hirb.rb - 0755 0755 @@ -88,5 +88,44 @@ *.dylib + + + + ${project.basedir}/../hbase-server/target/ + lib + + ${server.test.jar} + + 0644 + + + ${project.basedir}/../hbase-mapreduce/target/ + lib + + ${mapreduce.test.jar} + + 0644 + + + ${project.basedir}/../hbase-common/target/ + lib + + ${common.test.jar} + + 0644 + + + ${project.basedir}/../hbase-zookeeper/target/ + lib + + ${zookeeper.test.jar} + + 0644 + diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml index 79519613c3..c875b9534f 100644 --- a/hbase-assembly/src/main/assembly/client.xml +++ b/hbase-assembly/src/main/assembly/client.xml @@ -31,73 +31,37 @@ src/main/assembly/client-components.xml + true - - org.apache.hbase:hbase-annotations - org.apache.hbase:hbase-client - org.apache.hbase:hbase-common - org.apache.hbase:hbase-hadoop-compat - org.apache.hbase:hbase-hadoop2-compat - org.apache.hbase:hbase-mapreduce - org.apache.hbase:hbase-metrics - org.apache.hbase:hbase-metrics-api - org.apache.hbase:hbase-procedure - org.apache.hbase:hbase-protocol - org.apache.hbase:hbase-protocol-shaded - org.apache.hbase:hbase-server - org.apache.hbase:hbase-zookeeper + org.apache.hbase:hbase-shell - - lib false - - - + lib + + + + org.jruby:jruby-complete com.sun.jersey:* com.sun.jersey.contribs:* - - org.apache.hbase:hbase-external-blockcache - org.apache.hbase:hbase-http - org.apache.hbase:hbase-replication - org.apache.hbase:hbase-rest - org.apache.hbase:hbase-rsgroup - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - org.apache.hbase:hbase-shell - org.apache.hbase:hbase-thrift - org.jruby:jruby-complete + jline:jline + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-client-byo-hadoop + org.apache.hbase:hbase-shaded-mapreduce + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 - - - - - - true - - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - - - shaded-lib - false - - - - - org.apache.hbase:hbase-shaded-client - org.apache.hbase:hbase-shaded-mapreduce - - false - - + + @@ -123,15 +87,66 @@ + + + lib/shaded-clients + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + - lib/ruby org.jruby:jruby-complete + + + lib/client-facing-thirdparty + true + + + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 59502d67b4..2eb16e7649 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -37,7 +37,8 @@ ${project.basedir}/.. . - CHANGES.txt + CHANGES.md + RELEASENOTES.md README.txt 0644 diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index 69a800bc2a..72437839df 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -66,18 +66,51 @@ lib false - + org.jruby:jruby-complete com.sun.jersey:* com.sun.jersey.contribs:* jline:jline + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-client-byo-hadoop + org.apache.hbase:hbase-shaded-mapreduce + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + + true + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + + lib/shaded-clients + false + + + + org.apache.hbase:hbase-shaded-client + org.apache.hbase:hbase-shaded-mapreduce + org.apache.hbase:hbase-shaded-client-byo-hadoop + + + + + @@ -101,15 +134,56 @@ - + lib/ruby org.jruby:jruby-complete + + + lib/client-facing-thirdparty + true + + + com.github.stephenc.findbugs:findbugs-annotations + commons-logging:commons-logging + log4j:log4j + org.apache.htrace:htrace-core4 + org.apache.htrace:htrace-core + org.apache.yetus:audience-annotations + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + lib/zkcli diff --git a/pom.xml b/pom.xml index b9a1b7725e..ea02fe8c3c 100755 --- a/pom.xml +++ b/pom.xml @@ -1555,6 +1555,7 @@ hbase-annotations-${project.version}-tests.jar hbase-rsgroup-${project.version}-tests.jar hbase-mapreduce-${project.version}-tests.jar + hbase-zookeeper-${project.version}-tests.jar bash 2.21.0 surefire-junit47 @@ -1858,6 +1859,11 @@ hbase-shaded-client ${project.version} + + org.apache.hbase + hbase-shaded-client-byo-hadoop + ${project.version} + org.apache.hbase hbase-shaded-mapreduce -- 2.16.1 From ce717e6532b9f98b030ba8fa8777772ac8033010 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 1 May 2018 14:28:52 -0500 Subject: [PATCH 5/5] HBASE-20334 add a test that verifies basic client and MR integration --- dev-support/Jenkinsfile | 233 +++++++--- .../hbase_nightly_pseudo-distributed-test.sh | 516 +++++++++++++++++++++ dev-support/hbase_nightly_source-artifact.sh | 14 +- .../cache-apache-project-artifact.sh | 131 ++++++ 4 files changed, 836 insertions(+), 58 deletions(-) create mode 100755 dev-support/hbase_nightly_pseudo-distributed-test.sh create mode 100755 dev-support/jenkins-scripts/cache-apache-project-artifact.sh diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 2311e3552c..59d3227f58 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -60,54 +60,109 @@ pipeline { booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') } stages { - stage ('yetus install') { + stage ('scm-checkout') { steps { - sh '''#!/usr/bin/env bash -set -e -echo "Ensure we have a copy of Apache Yetus." -if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then - YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" - echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'" - if [ ! -d "${YETUS_DIR}" ]; then - echo "New download of Apache Yetus version ${YETUS_RELEASE}." - rm -rf "${WORKSPACE}/.gpg" - mkdir -p "${WORKSPACE}/.gpg" - chmod -R 700 "${WORKSPACE}/.gpg" - - echo "install yetus project KEYS" - curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS - gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS" - - echo "download yetus release ${YETUS_RELEASE}" - curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz" - curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc" - echo "verifying yetus release" - gpg --homedir "${WORKSPACE}/.gpg" --verify "yetus-${YETUS_RELEASE}-bin.tar.gz.asc" - mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz - else - echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}." - fi -else - YETUS_DIR="${WORKSPACE}/yetus-git" - rm -rf "${YETUS_DIR}" - echo "downloading from github" - curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz -fi -if [ ! -d "${YETUS_DIR}" ]; then - echo "unpacking yetus into '${YETUS_DIR}'" - mkdir -p "${YETUS_DIR}" - gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1 -fi - ''' - // Set up the file we need at PERSONALITY_FILE location - dir ("tools") { - sh """#!/usr/bin/env bash -set -e -echo "Downloading Project personality." -curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" - """ + dir('component') { + checkout scm + } + } + } + stage ('thirdparty installs') { + parallel { + stage ('yetus install') { + steps { + // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( + dir('downloads-yetus') { + // can't just do a simple echo or the directory won't be created. :( + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" +''' + } + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Apache Yetus." + if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then + YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" + echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'" + if [ ! -d "${YETUS_DIR}" ]; then + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-yetus" \ + --keys 'https://www.apache.org/dist/yetus/KEYS' \ + "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ + "yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz" + mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz + else + echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}." + fi + else + YETUS_DIR="${WORKSPACE}/yetus-git" + rm -rf "${YETUS_DIR}" + echo "downloading from github" + curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz + fi + if [ ! -d "${YETUS_DIR}" ]; then + echo "unpacking yetus into '${YETUS_DIR}'" + mkdir -p "${YETUS_DIR}" + gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1 + fi + ''' + // Set up the file we need at PERSONALITY_FILE location + dir ("tools") { + sh """#!/usr/bin/env bash + set -e + echo "Downloading Project personality." + curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" + """ + } + stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" + } + } + stage ('hadoop 2 cache') { + environment { + HADOOP2_VERSION="2.7.1" + } + steps { + // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( + dir('downloads-hadoop-2') { + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" +''' + } + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-hadoop-2" \ + --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \ + "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz" + ''' + stash name: 'hadoop-2', includes: "hadoop-${HADOOP2_VERSION}-bin.tar.gz" + } + } + stage ('hadoop 3 cache') { + environment { + HADOOP3_VERSION="3.0.0" + } + steps { + // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( + dir('downloads-hadoop-3') { + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" +''' + } + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-hadoop-3" \ + --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ + "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" + ''' + stash name: 'hadoop-3', includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz" + } } - stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" } } stage ('init health results') { @@ -149,6 +204,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" ''' unstash 'yetus' + // since we have a new node definition we need to re-do the scm checkout dir('component') { checkout scm } @@ -441,7 +497,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" } // This is meant to mimic what a release manager will do to create RCs. // See http://hbase.apache.org/book.html#maven.release - stage ('create source tarball') { + stage ('packaging and integration') { tools { maven 'Maven (latest)' // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in. @@ -454,14 +510,17 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" sh '''#!/bin/bash -e echo "Setting up directories" rm -rf "output-srctarball" && mkdir "output-srctarball" + rm -rf "output-integration" && mkdir "output-integration" "output-integration/hadoop-2" "output-integration/hadoop-3" "output-integration/hadoop-3-shaded" rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball" + rm -rf "hbase-install" && mkdir "hbase-install" + rm -rf "hbase-client" && mkdir "hbase-client" + rm -rf "hadoop-2" && mkdir "hadoop-2" + rm -rf "hadoop-3" && mkdir "hadoop-3" rm -rf ".m2-for-repo" && mkdir ".m2-for-repo" rm -rf ".m2-for-src" && mkdir ".m2-for-src" - echo '(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console].' >output-srctarball/commentfile + echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile + echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile ''' - dir('component') { - checkout scm - } sh '''#!/usr/bin/env bash set -e rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" @@ -470,6 +529,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" ls -lh "output-srctarball/machine" ''' sh """#!/bin/bash -e + echo "Checking the steps for an RM to make a source artifact, then a binary artifact." if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \ --intermediate-file-dir output-srctarball \ --unpack-temp-dir unpacked_src_tarball \ @@ -483,12 +543,78 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" exit 1 fi """ + echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'" + sh '''#!/bin/bash -e + if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then + echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile + exit 1 + fi + install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | head -n 1) + tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install" + client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1) + tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client" +''' + unstash 'hadoop-2' + echo "Attempting to use run an instance on top of Hadoop 2." + sh '''#!/bin/bash -xe + artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1) + tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2" + if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ + --single-process \ + --working-dir output-integration/hadoop-2 \ + --hbase-client-install "hbase-client" \ + "hbase-install" \ + "hadoop-2/bin/hadoop" \ + hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + >output-integration/hadoop-2.log 2>&1 ; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile + exit 2 + fi +''' + unstash 'hadoop-3' + echo "Attempting to use run an instance on top of Hadoop 3." + sh '''#!/bin/bash -e + artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) + tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" + if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ + --single-process \ + --working-dir output-integration/hadoop-3 \ + --hbase-client-install hbase-client \ + hbase-install \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + >output-integration/hadoop-3.log 2>&1 ; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile + exit 2 + fi + echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program." + if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ + --single-process \ + --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ + --working-dir output-integration/hadoop-3-shaded \ + --hbase-client-install hbase-client \ + hbase-install \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + >output-integration/hadoop-3-shaded.log 2>&1 ; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile + exit 2 + fi + echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile +''' + + } post { always { - stash name: 'srctarball-result', includes: "output-srctarball/commentfile" + stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile" archive 'output-srctarball/*' archive 'output-srctarball/**/*' + archive 'output-integration/*' + archive 'output-integration/**/*' } } } @@ -509,7 +635,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" "${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile", "${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile", "${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile", - 'output-srctarball/commentfile'] + 'output-srctarball/commentfile', + 'output-integration/commentfile'] echo env.BRANCH_NAME echo env.BUILD_URL echo currentBuild.result diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh new file mode 100755 index 0000000000..cc2dd5ec4e --- /dev/null +++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh @@ -0,0 +1,516 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +function usage { + echo "Usage: ${0} [options] /path/to/component/bin-install /path/to/hadoop/executable /path/to/hadoop/hadoop-yarn-server-tests-tests.jar /path/to/hadoop/hadoop-mapreduce-client-jobclient-tests.jar" + echo "" + echo " --zookeeper-data /path/to/use Where the embedded zookeeper instance should write its data." + echo " defaults to 'zk-data' in the working-dir." + echo " --working-dir /path/to/use Path for writing configs and logs. must exist." + echo " defaults to making a directory via mktemp." + echo " --hadoop-client-classpath /path/to/some.jar:/path/to/another.jar classpath for hadoop jars." + echo " defaults to 'hadoop classpath'" + echo " --hbase-client-install /path/to/unpacked/client/tarball if given we'll look here for hbase client jars instead of the bin-install" + echo " --force-data-clean Delete all data in HDFS and ZK prior to starting up hbase" + echo " --single-process Run as single process instead of pseudo-distributed" + echo "" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 4 ]; then + usage +fi + +# Get arguments +declare component_install +declare hadoop_exec +declare working_dir +declare zk_data_dir +declare clean +declare distributed="true" +declare hadoop_jars +declare hbase_client +while [ $# -gt 0 ] +do + case "$1" in + --working-dir) shift; working_dir=$1; shift;; + --force-data-clean) shift; clean="true";; + --zookeeper-data) shift; zk_data_dir=$1; shift;; + --single-process) shift; distributed="false";; + --hadoop-client-classpath) shift; hadoop_jars="$1"; shift;; + --hbase-client-install) shift; hbase_client="$1"; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have where component checkout is. +if [ $# -lt 4 ]; then + usage +fi +component_install="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" +hadoop_exec="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")" +yarn_server_tests_test_jar="$(cd "$(dirname "$3")"; pwd)/$(basename "$3")" +mapred_jobclient_test_jar="$(cd "$(dirname "$4")"; pwd)/$(basename "$4")" + +if [ ! -x "${hadoop_exec}" ]; then + echo "hadoop cli does not appear to be executable." >&2 + exit 1 +fi + +if [ ! -d "${component_install}" ]; then + echo "Path to HBase binary install should be a directory." >&2 + exit 1 +fi + +if [ ! -f "${yarn_server_tests_test_jar}" ]; then + echo "Specified YARN server tests test jar is not a file." >&2 + exit 1 +fi + +if [ ! -f "${mapred_jobclient_test_jar}" ]; then + echo "Specified MapReduce jobclient test jar is not a file." >&2 + exit 1 +fi + +if [ -z "${working_dir}" ]; then + if ! working_dir="$(mktemp -d -t hbase-pseudo-dist-test)" ; then + echo "Failed to create temporary working directory. Please specify via --working-dir" >&2 + exit 1 + fi +else + # absolutes please + working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")" + if [ ! -d "${working_dir}" ]; then + echo "passed working directory '${working_dir}' must already exist." >&2 + exit 1 + fi +fi + +if [ -z "${zk_data_dir}" ]; then + zk_data_dir="${working_dir}/zk-data" + mkdir "${zk_data_dir}" +else + # absolutes please + zk_data_dir="$(cd "$(dirname "${zk_data_dir}")"; pwd)/$(basename "${zk_data_dir}")" + if [ ! -d "${zk_data_dir}" ]; then + echo "passed directory for unpacking the source tarball '${zk_data_dir}' must already exist." + exit 1 + fi +fi + +if [ -z "${hbase_client}" ]; then + hbase_client="${component_install}" +else + echo "Using HBase client-side artifact" + # absolutes please + hbase_client="$(cd "$(dirname "${hbase_client}")"; pwd)/$(basename "${hbase_client}")" + if [ ! -d "${hbase_client}" ]; then + echo "If given hbase client install should be a directory with contents of the client tarball." >&2 + exit 1 + fi +fi + +if [ -n "${hadoop_jars}" ]; then + declare -a tmp_jars + for entry in $(echo "${hadoop_jars}" | tr ':' '\n'); do + tmp_jars=("${tmp_jars[@]}" "$(cd "$(dirname "${entry}")"; pwd)/$(basename "${entry}")") + done + hadoop_jars="$(IFS=:; echo "${tmp_jars[*]}")" +fi + + +echo "You'll find logs and temp files in ${working_dir}" + +function redirect_and_run { + log_base=$1 + shift + echo "$*" >"${log_base}.err" + "$@" >"${log_base}.out" 2>>"${log_base}.err" +} + +(cd "${working_dir}" + +echo "Hadoop version information:" +"${hadoop_exec}" version +hadoop_version=$("${hadoop_exec}" version | head -n 1) +hadoop_version="${hadoop_version#Hadoop }" +if [ "${hadoop_version%.*.*}" -gt 2 ]; then + "${hadoop_exec}" envvars +else + echo "JAVA_HOME: ${JAVA_HOME}" +fi + +# Ensure that if some other Hadoop install happens to be present in the environment we ignore it. +HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true" +export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP + +if [ -n "${clean}" ]; then + echo "Cleaning out ZooKeeper..." + rm -rf "${zk_data_dir:?}/*" +fi + +echo "HBase version information:" +"${component_install}/bin/hbase" version 2>/dev/null +hbase_version=$("${component_install}/bin/hbase" version | head -n 1 2>/dev/null) +hbase_version="${hbase_version#HBase }" + +if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" ]; then + echo "HBase binary install doesn't appear to include a shaded mapreduce artifact." >&2 + exit 1 +fi + +if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-${hbase_version}.jar" ]; then + echo "HBase binary install doesn't appear to include a shaded client artifact." >&2 + exit 1 +fi + +if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar" ]; then + echo "HBase binary install doesn't appear to include a shaded client artifact." >&2 + exit 1 +fi + +echo "Writing out configuration for HBase." +rm -rf "${working_dir}/hbase-conf" +mkdir "${working_dir}/hbase-conf" + +if [ -f "${component_install}/conf/log4j.properties" ]; then + cp "${component_install}/conf/log4j.properties" "${working_dir}/hbase-conf/log4j.properties" +else + cat >"${working_dir}/hbase-conf/log4j.properties" <"${working_dir}/hbase-conf/hbase-site.xml" < + + + + + hbase.rootdir + + /hbase + + + hbase.zookeeper.property.dataDir + ${zk_data_dir} + + + hbase.cluster.distributed + ${distributed} + + +EOF + +if [ "true" = "${distributed}" ]; then + cat >"${working_dir}/hbase-conf/regionservers" <"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" & +echo "$!" > "${working_dir}/hadoop.pid" + +sleep_time=2 +until [ -s "${working_dir}/hbase-conf/core-site.xml" ]; do + printf '\twaiting for Hadoop to finish starting up.\n' + sleep "${sleep_time}" + sleep_time="$((sleep_time*2))" +done + +if [ "${hadoop_version%.*.*}" -gt 2 ]; then + echo "Verifying configs" + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest +fi + +if [ -n "${clean}" ]; then + echo "Cleaning out HDFS..." + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r /hbase + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example/ + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example-region-listing.data +fi + +echo "Listing HDFS contents" +redirect_and_run "${working_dir}/hadoop_cluster_smoke" \ + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R / + +echo "Starting up HBase" +HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh" + +sleep_time=2 +until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 <"${working_dir}/table_create.log" 2>&1 < 1000, SPLITALGO => 'UniformSplit'} +EOF + +echo "writing out example TSV to example.tsv" +cat >"${working_dir}/example.tsv" <"${working_dir}/scan_import.out" 2>"${working_dir}/scan_import.err" </dev/null | tail -n 1) +if [ ! "${import_rowcount}" -eq 48 ]; then + echo "ERROR: Instead of finding 48 rows, we found ${import_rowcount}." + exit 2 +fi + +if [ -z "${hadoop_jars}" ]; then + echo "Hadoop client jars not given; getting them from 'hadoop classpath' for the example." + hadoop_jars=$("${hadoop_exec}" --config "${working_dir}/hbase-conf/" classpath) +fi + +echo "Building shaded client example." +cat >"${working_dir}/HBaseClientReadWriteExample.java" < regions = new LinkedList<>(); + try (Admin admin = connection.getAdmin()) { + final ClusterMetrics cluster = admin.getClusterMetrics(); + System.out.println(String.format("\tCluster reports version %s, ave load %f, region count %d", cluster.getHBaseVersion(), cluster.getAverageLoad(), cluster.getRegionCount())); + for (ServerMetrics server : cluster.getLiveServerMetrics().values()) { + for (RegionMetrics region : server.getRegionMetrics().values()) { + regions.add(region.getNameAsString()); + } + } + } + final Path listing = new Path("example-region-listing.data"); + System.out.println("Writing list to HDFS"); + try (FileSystem fs = FileSystem.newInstance(hadoop)) { + final Path path = fs.makeQualified(listing); + try (FSDataOutputStream out = fs.create(path)) { + out.writeInt(regions.size()); + for (String region : regions) { + out.writeUTF(region); + } + out.hsync(); + } + } + final List puts = new LinkedList<>(); + final Put marker = new Put(new byte[] { (byte)0 }); + System.out.println("Reading list from HDFS"); + try (FileSystem fs = FileSystem.newInstance(hadoop)) { + final Path path = fs.makeQualified(listing); + final CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); + try (FSDataInputStream in = fs.open(path)) { + final int count = in.readInt(); + marker.addColumn(FAMILY_BYTES, Bytes.toBytes("count"), Bytes.toBytes(count)); + for(int i = 0; i < count; i++) { + builder.clear(); + final byte[] row = Bytes.toBytes(in.readUTF()); + final Put put = new Put(row); + builder.setRow(row); + builder.setFamily(FAMILY_BYTES); + builder.setType(Cell.Type.Put); + put.add(builder.build()); + puts.add(put); + } + } + } + System.out.println("Writing list into HBase table"); + try (Table table = connection.getTable(TableName.valueOf("test:example"))) { + table.put(marker); + table.put(puts); + } + } + } +} +EOF +redirect_and_run "${working_dir}/hbase-shaded-client-compile" \ + javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java" +echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table." +# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190. +redirect_and_run "${working_dir}/hbase-shaded-client-example" \ + java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample + +echo "Checking on results of example program." +"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data" + +"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_example.out" 2>"${working_dir}/scan_example.err" </dev/null | tail -n 1) +if [ "${example_rowcount}" -gt "1050" ]; then + echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/root regions, and 1 for example's count record" +else + echo "ERROR: Only found ${example_rowcount} rows." +fi + +) diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh index 945832498e..c435c4882f 100755 --- a/dev-support/hbase_nightly_source-artifact.sh +++ b/dev-support/hbase_nightly_source-artifact.sh @@ -174,9 +174,13 @@ cd "${unpack_dir}" echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball." if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \ assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then - echo "Building a binary tarball from the source tarball succeeded." -else - echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details." - exit 1 + for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do + if [ -f "${artifact}" ]; then + # TODO check the layout of the binary artifact we just made. + echo "Building a binary tarball from the source tarball succeeded." + exit 0 + fi + done fi -# TODO check the layout of the binary artifact we just made. +echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details." +exit 1 diff --git a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh new file mode 100755 index 0000000000..57853c3d09 --- /dev/null +++ b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +function usage { + echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz" + echo "" + echo " --force for a redownload even if /path/to/download/file.tar.gz exists." + echo " --working-dir /path/to/use Path for writing tempfiles. must exist." + echo " defaults to making a directory via mktemp that we clean." + echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download." + echo "" + exit 1 +} +# if no args specified, show usage +if [ $# -lt 2 ]; then + usage +fi + + +# Get arguments +declare done_if_cached="true" +declare working_dir +declare cleanup="true" +declare keys +while [ $# -gt 0 ] +do + case "$1" in + --force) shift; done_if_cached="false";; + --working-dir) shift; working_dir=$1; cleanup="false"; shift;; + --keys) shift; keys=$1; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have required args +if [ $# -lt 2 ]; then + usage +fi + +target="$1" +artifact="$2" + +if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then + echo "Reusing existing download of '${artifact}'." + exit 0 +fi + +if [ -z "${working_dir}" ]; then + if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then + echo "Failed to create temporary working directory. Please specify via --working-dir" >&2 + exit 1 + fi +else + # absolutes please + working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")" + if [ ! -d "${working_dir}" ]; then + echo "passed working directory '${working_dir}' must already exist." >&2 + exit 1 + fi +fi + +function cleanup { + if [ "true" = "${cleanup}" ]; then + echo "cleaning up temp space." + rm -rf "${working_dir}" + fi +} +trap cleanup EXIT SIGQUIT + +echo "New download of '${artifact}'" + +# N.B. this comes first so that if gpg falls over we skip the expensive download. +if [ -n "${keys}" ]; then + if [ ! -d "${working_dir}/.gpg" ]; then + rm -rf "${working_dir}/.gpg" + mkdir -p "${working_dir}/.gpg" + chmod -R 700 "${working_dir}/.gpg" + fi + + echo "installing project KEYS" + curl -L --fail -o "${working_dir}/KEYS" "${keys}" + if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then + echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2 + echo " gpg: can't connect to the agent: File name too long" >&2 + # we mean to give them the command to run, not to run it. + #shellcheck disable=SC2016 + echo 'then you prolly need to create /var/run/user/$(id -u)' >&2 + echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2 + exit 2 + fi + + echo "downloading signature" + curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc" +fi + +echo "downloading artifact" +if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then + echo "Artifact wasn't in mirror system. falling back to archive.a.o." + curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}" +fi + +if [ -n "${keys}" ]; then + echo "verifying artifact signature" + gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc" + echo "signature good." +fi + +echo "moving artifact into place at '${target}'" +# ensure we're on the same filesystem +mv "${working_dir}/artifact" "${target}.copying" +# attempt atomic move +mv "${target}.copying" "${target}" +echo "all done!" -- 2.16.1