From be78fb244b4cd79151b9ba1dc78dfa423b8a3acf Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 26 Feb 2018 11:36:44 -0600 Subject: [PATCH] HBASE-20070 refactor website generation script * rely on git plumbing commands when checking if we've build the site for a particular commit already * switch to presuming we were invoked via bash -e * only export JAVA/MAVEN HOME if they aren't already set. * add some docs about assumptions --- .../jenkins-scripts/generate-hbase-website.sh | 102 ++++++++++++--------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh index 06d160a64c..23ca086a80 100644 --- a/dev-support/jenkins-scripts/generate-hbase-website.sh +++ b/dev-support/jenkins-scripts/generate-hbase-website.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # #/** # * Licensed to the Apache Software Foundation (ASF) under one @@ -23,7 +23,12 @@ # # It needs to be built on a Jenkins server with the label git-websites # -# It expects to have the hbase repo cloned to the directory hbase +# It presumes it can rely on environment variables: +# * WORKSPACE - a working directory it's free to write to +# +# It expects to have the hbase repo cloned to the directory 'hbase' within ${WORKSPACE} +# +# It expects to be invoked as given in the shebang line above (i.e. with 'fail on error' set) # # If there is a build error, the Jenkins job is configured to send an email @@ -36,60 +41,70 @@ mkdir -p "${LOCAL_REPO}" rm -Rf -- *.patch *.patch.zip hbase/target target *.txt hbase-site # Set up the environment -export JAVA_HOME=$JDK_1_8_LATEST__HOME -export PATH=$JAVA_HOME/bin:$MAVEN_3_3_3_HOME/bin:$PATH +if [ -z "${JAVA_HOME}" ]; then + JAVA_HOME="${JDK_1_8_LATEST__HOME}" + export JAVA_HOME + export PATH="${JAVA_HOME}/bin:${PATH}" +fi +if [ -z "${MAVEN_HOME}" ]; then + MAVEN_HOME="${MAVEN_3_3_3_HOME}" + export MAVEN_HOME + export PATH="${MAVEN_HOME}/bin:${PATH}" +fi export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${LOCAL_REPO}" # Verify the Maven version mvn -version +# Verify the git version +git --version # Save and print the SHA we are building -CURRENT_HBASE_COMMIT="$(git log --pretty=format:%H -n1)" +CURRENT_HBASE_COMMIT="$(cd hbase && git show-ref --hash --dereference --verify refs/remotes/origin/HEAD)" +# Fail if it's empty +[ -n "${CURRENT_HBASE_COMMIT}" ] echo "Current HBase commit: $CURRENT_HBASE_COMMIT" # Clone the hbase-site repo manually so it doesn't trigger spurious # commits in Jenkins. git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git -# Figure out the last commit we built the site from, and bail if the build -# still represents the SHA of HBase master -cd "${WORKSPACE}/hbase-site" || exit -1 -git log --pretty=%s | grep ${CURRENT_HBASE_COMMIT} -PUSHED=$? -echo "PUSHED is $PUSHED" +# Figure out if the commit of the hbase repo has already been built and bail if so. +declare -i PUSHED +PUSHED=$(cd hbase-site && git rev-list --grep "${CURRENT_HBASE_COMMIT}" --fixed-strings --count HEAD) +echo "[DEBUG] hash was found in $PUSHED commits for hbase-site repository." -if [ $PUSHED -eq 0 ]; then +if [ "${PUSHED}" -ne 0 ]; then echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building." exit 0 else - echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet. $PUSHED" + echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet." fi # Go to the hbase directory so we can build the site -cd "${WORKSPACE}/hbase" || exit -1 +cd "${WORKSPACE}/hbase" # This will only be set for builds that are triggered by SCM change, not manual builds -if [ "$CHANGE_ID" ]; then +if [ -n "$CHANGE_ID" ]; then echo -n " ($CHANGE_ID - $CHANGE_TITLE)" fi # Build and install HBase, then build the site echo "Building HBase" -mvn \ - -DskipTests \ - -Dmaven.javadoc.skip=true \ - --batch-mode \ - -Dcheckstyle.skip=true \ - -Dfindbugs.skip=true \ - --log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \ - clean install \ -&& mvn clean site \ - --batch-mode \ - -DskipTests \ - --log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" - -status=$? -if [ $status -ne 0 ]; then +if mvn \ + -DskipTests \ + -Dmaven.javadoc.skip=true \ + --batch-mode \ + -Dcheckstyle.skip=true \ + -Dfindbugs.skip=true \ + --log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \ + clean install \ + && mvn clean site \ + --batch-mode \ + -DskipTests \ + --log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt"; then + echo "Successfully built site." +else + status=$? echo "Failure: mvn clean site" exit $status fi @@ -107,7 +122,7 @@ if [ $status -ne 0 ] || [ ! -d target/staging ]; then fi # Get ready to update the hbase-site repo with the new artifacts -cd "${WORKSPACE}/hbase-site" || exit -1 +cd "${WORKSPACE}/hbase-site" #Remove previously-generated files FILES_TO_REMOVE=("hbase-*" @@ -125,11 +140,14 @@ FILES_TO_REMOVE=("hbase-*" "images") for FILE in "${FILES_TO_REMOVE[@]}"; do - echo "Removing ${WORKSPACE}/hbase-site/$FILE" - rm -Rf "${FILE}" + if [ -e "${FILE}" ]; then + echo "Removing ${WORKSPACE}/hbase-site/$FILE" + rm -Rf "${FILE}" + fi done # Copy in the newly-built artifacts +# TODO what do we do when the site build wants to remove something? Can't rsync because e.g. release-specific docs. cp -au "${WORKSPACE}"/hbase/target/staging/* . # If the index.html is missing, bail because this is serious @@ -149,22 +167,22 @@ else git diff --name-status --diff-filter=ADCRTXUB origin/asf-site # Create a patch, which Jenkins can save as an artifact and can be examined for debugging git format-patch --stdout origin/asf-site > "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch" + if [ ! -s "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch" ]; then + echo "Something went wrong when creating the patch of our updated site." + exit 1 + fi echo "Change set saved to patch ${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch" # Push the real commit - git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1) + #git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1) + echo "Didn't actually push" # Create an empty commit to work around INFRA-10751 git commit --allow-empty -m "INFRA-10751 Empty commit" # Push the empty commit - git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1) + # git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1) + echo "Didn't actually push" echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes." - git fetch origin - git reset --hard origin/asf-site # Zip up the patch so Jenkins can save it - cd "${WORKSPACE}" || exit -1 + cd "${WORKSPACE}" zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch" fi - -#echo "Dumping current environment:" -#env - -- 2.16.1