From 30cb8717aaef48bb79e64fce23e1b6b653813195 Mon Sep 17 00:00:00 2001 From: TAK LON WU Date: Sat, 23 Feb 2019 11:43:47 -0800 Subject: [PATCH] HBASE-21963 Add a script for building and verifying release candidate --- dev-support/hbase-vote.sh | 168 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 dev-support/hbase-vote.sh diff --git a/dev-support/hbase-vote.sh b/dev-support/hbase-vote.sh new file mode 100644 index 0000000..6b3bda3 --- /dev/null +++ b/dev-support/hbase-vote.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +usage() { + SCRIPT=$(basename "${BASH_SOURCE[@]}") + + cat << __EOF +hbase-vote. A script for standard vote which verifies the following items +1. Checksum of sources and binaries +2. Signature of sources and binaries +3. Rat check +4. Built from source +5. Unit tests + +Usage: ${SCRIPT} -s | --source [-k | --key ] [-f | --keys-file-url ] + ${SCRIPT} -h | --help + + -h | --help Show this screen. + -s | --source '' A URL pointing to the release candidate sources and binaries + e.g. https://dist.apache.org/repos/dist/dev/hbase/hbase-RC0/ + -k | --key '' A signature of the public key, e.g. 9AD2AE49 + -f | --keys-file-url '' the URL of the key file, default is + http://www.apache.org/dist/hbase/KEYS +__EOF +} + +while ((${#})); do + case "${1}" in + -h | --help ) + usage; exit 0 ;; + -s | --source ) + SOURCE_URL="${2}"; shift 2 ;; + -k | --key ) + SIGNING_KEY="${2}"; shift 2 ;; + -f | --keys-file-url ) + KEY_FILE_URL="${2}"; shift 2 ;; + * ) + usage >&2; exit 1 ;; + esac +done + +# Source url must be provided +if [ -z "${SOURCE_URL}" ]; then + usage; + exit 1 +fi + +cat << __EOF +Although This tool helps verifying HBase RC build and unit tests, +operator may still consider to verify the following manually +1. Verify the compat-check-report.html +2. Any on cluster Integration test or performance test, e.g. LTT load 100M or + ITBLL 1B rows with serverKilling monkey +3. Other concerns if any +__EOF + +HBASE_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2) +HBASE_VERSION=$(echo "${HBASE_RC_VERSION}" | sed -e 's/RC[0-9]//g' | sed -e 's/hbase-//g') +JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g') + +# default value for verification targets, 0 = failed +CHECKSUM=0 +SIGNATURE=0 +RAT_CHECK=0 +BUILD_FROM_SOURCE=0 +UNIT_TEST=0 + +function download_and_import_keys() { + KEY_FILE_URL="${KEY_FILE_URL:-http://www.apache.org/dist/hbase/KEYS}" + echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}" + # download the keys file into file KEYS + wget -O KEYS "${KEY_FILE_URL}" + gpg --import KEYS + if [ -n "${SIGNING_KEY}" ]; then + gpg --list-keys "${SIGNING_KEY}" + fi +} + +function download_release_candidate () { + # get all files from release candidate repo + wget --recursive --no-parent "${SOURCE_URL}" +} + +function verify_signatures() { + gpg --verify hbase-"${HBASE_VERSION}"-bin.tar.gz.asc hbase-"${HBASE_VERSION}"-bin.tar.gz + gpg --verify hbase-"${HBASE_VERSION}"-src.tar.gz.asc hbase-"${HBASE_VERSION}"-src.tar.gz + SIGNATURE=1 +} + +function verify_checksums() { + gpg --print-md SHA512 hbase-"${HBASE_VERSION}"-src.tar.gz > src.sha512 + diff src.sha512 hbase-"${HBASE_VERSION}"-src.tar.gz.sha512 && CHECKSUM=1 || CHECKSUM=0 + rm -f src.sha512 + + gpg --print-md SHA512 hbase-"${HBASE_VERSION}"-bin.tar.gz > bin.sha512 + diff bin.sha512 hbase-"${HBASE_VERSION}"-bin.tar.gz.sha512 && CHECKSUM=1 || CHECKSUM=0 + rm -f bin.sha512 +} + +function unzip_from_source() { + tar -zxvf hbase-"${HBASE_VERSION}"-src.tar.gz + cd hbase-"${HBASE_VERSION}" +} + +function rat_test() { + mvn clean apache-rat:check && RAT_CHECK=1 +} + +function build_from_source() { + mvn clean install -DskipTests && BUILD_FROM_SOURCE=1 +} + +function run_all_tests() { + mvn test -P runAllTests && UNIT_TEST=1 +} + +function execute() { + ${1} || print_when_exit +} + +function print_when_exit() { + cat << __EOF + * Checksum : $( ((CHECKSUM)) && echo "ok" || echo "failed" ) + * Signature: $( ((SIGNATURE)) && echo "ok" || echo "failed" ) + * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK)) && echo "ok" || echo "failed" ) + - mvn clean apache-rat:check + * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE)) && echo "ok" || echo "failed" ) + - mvn clean install -DskipTests + * Unit tests pass (${JAVA_VERSION}): $( ((UNIT_TEST)) && echo "ok" || echo "failed" ) + - mvn test -P runAllTests +__EOF + if ((CHECKSUM)) && ((SIGNATURE)) && ((RAT_CHECK)) && ((BUILD_FROM_SOURCE)) && ((UNIT_TEST)) ; then + exit 0 + fi + exit 1 +} + +download_and_import_keys +download_release_candidate +pushd dist.apache.org/repos/dist/dev/hbase/"${HBASE_RC_VERSION}" + +execute verify_signatures +execute verify_checksums +execute unzip_from_source +execute rat_test +execute build_from_source +execute run_all_tests + +popd + +print_when_exit -- 2.10.1 (Apple Git-78)