From d52a3a84c7cbc45a8b976b470ca6e961710b4aaf Mon Sep 17 00:00:00 2001 From: stack Date: Wed, 11 Sep 2019 18:10:39 -0700 Subject: [PATCH] HBASE-23003 [HBCK2/hbase-operator-tools] Release-making scripts --- dev-support/create-release/README.txt | 17 +++++-- .../create-release/do-release-docker.sh | 6 +-- dev-support/create-release/do-release.sh | 12 ++--- dev-support/create-release/release-build.sh | 44 ++++++++++++------- dev-support/create-release/release-tag.sh | 8 ++-- dev-support/create-release/release-util.sh | 25 +++++------ 6 files changed, 67 insertions(+), 45 deletions(-) diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt index 2712653..f7fc722 100644 --- a/dev-support/create-release/README.txt +++ b/dev-support/create-release/README.txt @@ -1,12 +1,21 @@ -Entrance script is _do-release-docker.sh_. Requires a local docker; -for example, on mac os x, a _Docker for Desktop_ installed and -running. It does all steps building a release. +Entrance script is _do-release-docker.sh_. We run the release +creation inside a Docker container to ensure consistent +build environment (See _hbase_rm_ dir for the _Dockerfile_ +used). The script depends on their being a local docker; +for example, on mac os x, a _Docker for Desktop_ installed +and running. It does all steps building a release. + +_do-release-docker.sh_ calls _do-release.sh_ which is the core +absent docker machinations that runs the release-making process. +Use it to avoid running release creation in a container. If you just want to generate tarballs locally, run release-build.sh. It will ask for passwords to use building or you can supply as environment variables. For example: - $ PROJECT="hbase-operator-tools" ASF_USERNAME="USERNAME" ASF_PASSWORD="PASSWORD" GPG_PASSPHRASE="PASSWORD" GPG_KEY="stack@duboce.net" GIT_REF=e3eb8ce83075cdfee7f7a9be3cb9de7686c4905f ./release-build.sh build + $ ASF_USERNAME="USERNAME" ASF_PASSWORD="PASSWORD" GPG_PASSPHRASE="PASSWORD" GPG_KEY="stack@duboce.net" GIT_REF=e3eb8ce83075cdfee7f7a9be3cb9de7686c4905f ./release-build.sh build + +Ditto for running _do-release*.sh_. Before starting the RC build, run a reconciliation of what is in JIRA with what is in the commit log. Make sure they align and that diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 1e63486..c18014f 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -47,14 +47,12 @@ # 1. https://github.com/apache/spark/tree/master/dev/create-release # set -e -set -x export PROJECT="hbase-operator-tools" SELF=$(cd $(dirname $0) && pwd) . "$SELF/release-util.sh" - function usage { local NAME=$(basename $0) cat <\(.*\)<.*$/\1/p' pom.xml | grep -v '-assembly' | tr '\n' ' '` + pwd # Publish ${PROJECT} to Maven release repo echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" echo "Publish version is $VERSION" @@ -224,8 +232,8 @@ if [[ "$1" == "publish-release" ]]; then -Dmaven.repo.local=${tmp_repo} pushd $tmp_repo/org/apache/hbase # Remove any extra files generated during install - # Do find in hbase* because thirdparty is at same level! - find hbase* -type f | grep -v \.jar | grep -v \.pom | xargs rm + # Remove extaneous files from module subdirs + find $modules -type f | grep -v \.jar | grep -v \.pom | xargs rm # Using Nexus API documented here: # https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API @@ -240,7 +248,7 @@ if [[ "$1" == "publish-release" ]]; then fi # this must have .asc, and .sha1 - it really doesn't like anything else there - for file in $(find hbase* -type f) + for file in $(find $modules -type f) do if [[ "$file" == *.asc ]]; then continue @@ -256,19 +264,23 @@ if [[ "$1" == "publish-release" ]]; then # Available on Linux; cut to keep only hash md5sum $file | cut -f1 -d' ' > $file.md5 fi - sha1sum $file | cut -f1 -d' ' > $file.sha1 + if [ $(command -v sha1sum) ]; then + sha1sum $file | cut -f1 -d' ' > $file.sha1 + else + shasum $file | cut -f1 -d' ' > $file.sha1 + fi done if ! is_dry_run; then nexus_upload=$NEXUS_ROOT/deployByRepositoryId/$staged_repo_id echo "Uplading files to $nexus_upload" - for file in $(find hbase* -type f) + for file in $(find $modules -type f) do # strip leading ./ file_short=$(echo $file | sed -e "s/\.\///") dest_url="$nexus_upload/org/apache/hbase/$file_short" echo " Uploading $file to $dest_url" - curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $file_short $dest_url + curl -u "$ASF_USERNAME:$ASF_PASSWORD" --upload-file $file_short $dest_url done echo "Closing nexus staging repository" @@ -288,5 +300,5 @@ if [[ "$1" == "publish-release" ]]; then fi cd .. -rm -rf hbase +rm -rf ${PROJECT} echo "ERROR: expects to be called with 'install', 'publish-release' or 'publish-snapshot'" diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh index be1325c..98be22f 100755 --- a/dev-support/create-release/release-tag.sh +++ b/dev-support/create-release/release-tag.sh @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set +x # Tags release. Updates releasenotes and changes. SELF=$(cd $(dirname $0) && pwd) @@ -66,8 +65,11 @@ init_mvn rm -rf ${PROJECT} -ASF_HBASE_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" -git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_HBASE_REPO" -b $GIT_BRANCH +ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" +# Ugly! +encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") +encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") +git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH # NOTE: Here we are prepending project name on version for fetching # changes from the HBASE JIRA. It has issues for hbase, hbase-conectors, # hbase-operator-tools, etc. diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 711a587..f6f81bf 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -16,18 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set -x DRY_RUN=${DRY_RUN:-0} GPG="gpg --pinentry-mode loopback --no-tty --batch" -if [[ -z "${ASF_REPO}" ]]; then - ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git" -fi -if [[ -z "${ASF_REPO_WEBUI}" ]]; then - ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=${PROJECT}.git" -fi -if [[ -z "${ASF_GITHUB_REPO}" ]]; then - ASF_GITHUB_REPO="https://github.com/apache/${PROJECT}" -fi YETUS_VERSION=0.9.0 function error { @@ -108,6 +98,15 @@ function get_api_diff_version { # Get all branches that begin with 'branch-', the hbase convention for # release branches, sort them and then pop off the most recent. function get_release_info { + if [[ -z "${ASF_REPO}" ]]; then + ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git" + fi + if [[ -z "${ASF_REPO_WEBUI}" ]]; then + ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=${PROJECT}.git" + fi + if [[ -z "${ASF_GITHUB_REPO}" ]]; then + ASF_GITHUB_REPO="https://github.com/apache/${PROJECT}" + fi if [ -z "$GIT_BRANCH" ]; then # If no branch is specified, find out the latest branch from the repo. GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" | @@ -184,7 +183,7 @@ function get_release_info { GIT_REF=$(read_config "GIT_REF" "$GIT_REF") fi export GIT_REF - export HBASE_PACKAGE_VERSION="$RELEASE_TAG" + export PACKAGE_VERSION="$RELEASE_TAG" export API_DIFF_TAG=$(get_api_diff_version $RELEASE_VERSION) @@ -333,7 +332,7 @@ make_src_release() { rm -rf "${basename}-src*" cd "${project}" git clean -d -f -x - tgz="${basename}-src.tgz" + tgz="${basename}-src.tar.gz" git archive --format=tar.gz --output="../${tgz}" --prefix="${basename}/" "${GIT_REF:-master}" cd .. echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output "${tgz}.asc" \ @@ -375,7 +374,7 @@ make_binary_release() { MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings install assembly:single -DskipTests \ -Dcheckstyle.skip=true ${PUBLISH_PROFILES} -Dmaven.repo.local=${tmp_repo} - cp ./hbase-assembly/target/${basename}*-bin.tar.gz .. + cp ./${PROJECT}-assembly/target/${basename}*-bin.tar.gz .. cd .. for i in `ls ${basename}*-bin.tar.gz`; do echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output $i.asc --detach-sig $i -- 2.19.1