From bdbf40f915c786e32830db0b078c41d8e7ace59e Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 26 Feb 2018 11:36:44 -0600 Subject: [PATCH] HBASE-20070 refactor website generation * rely on git plumbing commands when checking if we've built the site for a particular commit already * switch to forcing '-e' for bash * add command line switches for: path to hbase, working directory, and publishing * only export JAVA/MAVEN HOME if they aren't already set. * add some docs about assumptions * Update javadoc plugin to consistently be version 3.0.0 * update use of cp command so it works both on linux and mac --- .../jenkins-scripts/generate-hbase-website.sh | 242 +++++++++++++++------ pom.xml | 15 +- 2 files changed, 180 insertions(+), 77 deletions(-) diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh index 06d160a64c..78ff8615ea 100644 --- a/dev-support/jenkins-scripts/generate-hbase-website.sh +++ b/dev-support/jenkins-scripts/generate-hbase-website.sh @@ -23,73 +23,163 @@ # # It needs to be built on a Jenkins server with the label git-websites # -# It expects to have the hbase repo cloned to the directory hbase +# Allows specifying options for working directory, maven repo, and publishing to git +# run with --help for usage. # # If there is a build error, the Jenkins job is configured to send an email -LOCAL_REPO=${WORKSPACE}/.m2/repo -# Nuke the local maven repo each time, to start with a known environment -rm -Rf "${LOCAL_REPO}" -mkdir -p "${LOCAL_REPO}" +declare CURRENT_HBASE_COMMIT +declare PUSHED +declare FILE +declare WEBSITE_COMMIT_MSG +declare -a FILES_TO_REMOVE -# Clean any leftover files in case we are reusing the workspace -rm -Rf -- *.patch *.patch.zip hbase/target target *.txt hbase-site +set -e +function usage { + echo "Usage: ${0} [options] /path/to/hbase/checkout" + echo "" + echo " --working-dir /path/to/use Path for writing logs and a local checkout of hbase-site repo." + echo " if given must exist." + echo " defaults to making a directory in /tmp." + echo " --local-repo /path/for/maven/.m2 Path for putting local maven repo." + echo " if given must exist." + echo " defaults to making a clean directory in --working-dir." + echo " --publish if given, will attempt to push results back to the hbase-site repo." + echo " --help show this usage message." + exit 1 +} +# if no args specified, show usage +if [ $# -lt 1 ]; then + usage +fi + +# Get arguments +declare component_dir +declare working_dir +declare local_repo +declare publish +while [ $# -gt 0 ] +do + case "$1" in + --working-dir) shift; working_dir=$1; shift;; + --local-repo) shift; local_repo=$1; shift;; + --publish) shift; publish="true";; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# should still have where component checkout is. +if [ $# -lt 1 ]; then + usage +fi +component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" + +if [ -z "${working_dir}" ]; then + working_dir=/tmp + while [[ -e ${working_dir} ]]; do + working_dir=/tmp/hbase-generate-website-${RANDOM}.${RANDOM} + done + mkdir "${working_dir}" +else + # absolutes please + working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")" + if [ ! -d "${working_dir}" ]; then + echo "passed working directory '${working_dir}' must already exist." + exit 1 + fi +fi + +echo "You'll find logs and temp files in ${working_dir}" + +if [ -z "${local_repo}" ]; then + local_repo="${working_dir}/.m2/repo" + # Nuke the local maven repo each time, to start with a known environment + rm -Rf "${local_repo}" + mkdir -p "${local_repo}" +else + # absolutes please + local_repo="$(cd "$(dirname "${local_repo}")"; pwd)/$(basename "${local_repo}")" + if [ ! -d "${local_repo}" ]; then + echo "passed directory for storing the maven repo '${local_repo}' must already exist." + exit 1 + fi +fi # Set up the environment -export JAVA_HOME=$JDK_1_8_LATEST__HOME -export PATH=$JAVA_HOME/bin:$MAVEN_3_3_3_HOME/bin:$PATH -export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${LOCAL_REPO}" +if [ -z "${JAVA_HOME}" ]; then + JAVA_HOME="${JDK_1_8_LATEST__HOME}" + export JAVA_HOME + export PATH="${JAVA_HOME}/bin:${PATH}" +fi +if [ -z "${MAVEN_HOME}" ]; then + MAVEN_HOME="${MAVEN_3_3_3_HOME}" + export MAVEN_HOME + export PATH="${MAVEN_HOME}/bin:${PATH}" +fi +export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${local_repo}" # Verify the Maven version mvn -version +# Verify the git version +git --version + +cd "${working_dir}" + +# Clean any leftover files in case we are reusing the workspace +rm -Rf -- *.patch *.patch.zip target *.txt hbase-site # Save and print the SHA we are building -CURRENT_HBASE_COMMIT="$(git log --pretty=format:%H -n1)" +CURRENT_HBASE_COMMIT="$(cd "${component_dir}" && git show-ref --hash --dereference --verify refs/remotes/origin/HEAD)" +# Fail if it's empty +if [ -z "${CURRENT_HBASE_COMMIT}" ]; then + echo "Got back a blank answer for the current HEAD on the remote hbase repository. failing." + exit 1 +fi echo "Current HBase commit: $CURRENT_HBASE_COMMIT" # Clone the hbase-site repo manually so it doesn't trigger spurious # commits in Jenkins. git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git -# Figure out the last commit we built the site from, and bail if the build -# still represents the SHA of HBase master -cd "${WORKSPACE}/hbase-site" || exit -1 -git log --pretty=%s | grep ${CURRENT_HBASE_COMMIT} -PUSHED=$? -echo "PUSHED is $PUSHED" +# Figure out if the commit of the hbase repo has already been built and bail if so. +declare -i PUSHED +PUSHED=$(cd hbase-site && git rev-list --grep "${CURRENT_HBASE_COMMIT}" --fixed-strings --count HEAD) +echo "[DEBUG] hash was found in $PUSHED commits for hbase-site repository." -if [ $PUSHED -eq 0 ]; then +if [ "${PUSHED}" -ne 0 ]; then echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building." exit 0 else - echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet. $PUSHED" + echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet." fi # Go to the hbase directory so we can build the site -cd "${WORKSPACE}/hbase" || exit -1 +cd "${component_dir}" # This will only be set for builds that are triggered by SCM change, not manual builds -if [ "$CHANGE_ID" ]; then +if [ -n "$CHANGE_ID" ]; then echo -n " ($CHANGE_ID - $CHANGE_TITLE)" fi # Build and install HBase, then build the site echo "Building HBase" -mvn \ - -DskipTests \ - -Dmaven.javadoc.skip=true \ - --batch-mode \ - -Dcheckstyle.skip=true \ - -Dfindbugs.skip=true \ - --log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \ - clean install \ -&& mvn clean site \ - --batch-mode \ - -DskipTests \ - --log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" - -status=$? -if [ $status -ne 0 ]; then +if mvn \ + -DskipTests \ + -Dmaven.javadoc.skip=true \ + --batch-mode \ + -Dcheckstyle.skip=true \ + -Dfindbugs.skip=true \ + --log-file="${working_dir}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \ + clean install \ + && mvn clean site \ + --batch-mode \ + -DskipTests \ + --log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt"; then + echo "Successfully built site." +else + status=$? echo "Failure: mvn clean site" exit $status fi @@ -98,7 +188,7 @@ fi echo "Staging HBase site" mvn \ --batch-mode \ - --log-file="${WORKSPACE}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \ + --log-file="${working_dir}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \ site:stage status=$? if [ $status -ne 0 ] || [ ! -d target/staging ]; then @@ -107,7 +197,7 @@ if [ $status -ne 0 ] || [ ! -d target/staging ]; then fi # Get ready to update the hbase-site repo with the new artifacts -cd "${WORKSPACE}/hbase-site" || exit -1 +cd "${working_dir}/hbase-site" #Remove previously-generated files FILES_TO_REMOVE=("hbase-*" @@ -125,46 +215,62 @@ FILES_TO_REMOVE=("hbase-*" "images") for FILE in "${FILES_TO_REMOVE[@]}"; do - echo "Removing ${WORKSPACE}/hbase-site/$FILE" - rm -Rf "${FILE}" + if [ -e "${FILE}" ]; then + echo "Removing hbase-site/$FILE" + rm -Rf "${FILE}" + fi done # Copy in the newly-built artifacts -cp -au "${WORKSPACE}"/hbase/target/staging/* . +# TODO what do we do when the site build wants to remove something? Can't rsync because e.g. release-specific docs. +cp -pPR "${component_dir}"/target/staging/* . # If the index.html is missing, bail because this is serious if [ ! -f index.html ]; then echo "The index.html is missing. Aborting." exit 1 -else - # Add all the changes - echo "Adding all the files we know about" - git add . - # Create the commit message and commit the changes - WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT." - echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG" - git commit -m "${WEBSITE_COMMIT_MSG}" -a - # Dump a little report - echo "This commit changed these files (excluding Modified files):" - git diff --name-status --diff-filter=ADCRTXUB origin/asf-site - # Create a patch, which Jenkins can save as an artifact and can be examined for debugging - git format-patch --stdout origin/asf-site > "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch" - echo "Change set saved to patch ${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch" - # Push the real commit - git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1) - # Create an empty commit to work around INFRA-10751 +fi + +# Add all the changes +echo "Adding all the files we know about" +git add . +# Create the commit message and commit the changes +WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT." +echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG" +git commit -m "${WEBSITE_COMMIT_MSG}" -a +# Dump a little report +echo "This commit changed these files (excluding Modified files):" +git diff --name-status --diff-filter=ADCRTXUB origin/asf-site +# Create a patch, which Jenkins can save as an artifact and can be examined for debugging +git format-patch --stdout origin/asf-site > "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" +if [ ! -s "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" ]; then + echo "Something went wrong when creating the patch of our updated site." + exit 1 +fi +echo "Change set saved to patch ${working_dir}/${CURRENT_HBASE_COMMIT}.patch" + +if [ -n "${publish}" ]; then + echo "Publishing changes to remote repo..." + if true; then + #if git push origin asf-site; then + "changes pushed." + else + echo "Failed to push to asf-site. Website not updated." + exit 1 + fi + echo "Sending empty commit to work around INFRA-10751." git commit --allow-empty -m "INFRA-10751 Empty commit" # Push the empty commit - git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1) + if true; then + #if git push origin asf-site; then + echo "empty commit pushed." + else + echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" + exit 1 + fi echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes." - git fetch origin - git reset --hard origin/asf-site - - # Zip up the patch so Jenkins can save it - cd "${WORKSPACE}" || exit -1 - zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch" fi -#echo "Dumping current environment:" -#env - +# Zip up the patch so Jenkins can save it +cd "${working_dir}" +zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch" diff --git a/pom.xml b/pom.xml index db8a78882b..99415cdc30 100755 --- a/pom.xml +++ b/pom.xml @@ -606,6 +606,7 @@ org.apache.maven.plugins maven-javadoc-plugin + ${maven.javadoc.version} org.apache.maven.plugins @@ -1474,6 +1475,7 @@ 2.10 2.5.2 3.0.2 + 3.0.0 1.2 2.15.2 3.0.0 @@ -1544,6 +1546,9 @@ this parameter by invoking mvn with -Dbuild.id=$BUILD_ID--> ${maven.build.timestamp} bash + + none @@ -2239,14 +2244,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - -Xdoclint:none - - org.codehaus.mojo findbugs-maven-plugin @@ -3486,7 +3483,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.0.0 + ${maven.javadoc.version} -- 2.16.1