diff --git .gitignore .gitignore index 8578a64..1a19ed4 100644 --- .gitignore +++ .gitignore @@ -31,3 +31,4 @@ itests/hive-blobstore/src/test/resources/blobstore-conf.xml .DS_Store *.rej *.orig +patchprocess diff --git dev-support/checkstyle_YETUS-484.sh dev-support/checkstyle_YETUS-484.sh new file mode 100644 index 0000000..e297a13 --- /dev/null +++ dev-support/checkstyle_YETUS-484.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type checkstyle + +CHECKSTYLE_TIMER=0 +CHECKSTYLE_GOAL_DEFAULT="checkstyle" +CHECKSTYLE_GOAL="${CHECKSTYLE_GOAL_DEFAULT}" +CHECKSTYLE_OPTIONS_DEFAULT="-Dcheckstyle.consoleOutput=true" +CHECKSTYLE_OPTIONS="${CHECKSTYLE_OPTIONS_DEFAULT}" + +function checkstyle_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi + fi +} + +## @description usage help for checkstyle +## @audience private +## @stability evolving +## @replaceable no +function checkstyle_usage +{ + yetus_add_option "--checkstyle-goal=" "Checkstyle maven plugin goal to use, 'check' and 'checkstyle' supported. Defaults to '${CHECKSTYLE_GOAL_DEFAULT}'." +} + +## @description parse checkstyle args +## @audience private +## @stability evolving +## @replaceable no +## @param arg +## @param .. +function checkstyle_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --checkstyle-goal=*) + CHECKSTYLE_GOAL=${i#*=} + case ${CHECKSTYLE_GOAL} in + check) + CHECKSTYLE_OPTIONS="-Dcheckstyle.consoleOutput=true -Dcheckstyle.failOnViolation=false" + ;; + checkstyle) + ;; + *) + yetus_error "Warning: checkstyle goal ${CHECKSTYLE_GOAL} not supported. It may have unexpected behavior" + ;; + esac + ;; + esac + done +} + +## @description initialize the checkstyle plug-in +## @audience private +## @stability evolving +## @replaceable no +function checkstyle_initialize +{ + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install checkstyle + fi +} + +## @description checkstyle plug-in specific difference calculator +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function checkstyle_calcdiffs +{ + declare orig=$1 + declare new=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, strip filenames:line: + # this keeps column: in an attempt to increase + # accuracy in case of multiple, repeated errors + # since the column number shouldn't change + # if the line of code hasn't been touched + # remove the numbers from the error message for comparing + # so if only the error message numbers change + # we do not report new error + # shellcheck disable=SC2016 + cut -f3- -d: "${orig}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print $1":"$2}' > "${tmp}.branch" + # shellcheck disable=SC2016 + cut -f3- -d: "${new}" | awk -F'\1' '{ gsub("[0-9,]+", "", $2) ;print $1":"$2}' > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # removing extra marker before the chekstyle error + # message which was needed for calculations + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 | tr -d $'\x01' + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +## @description execute checkstyle +## @audience private +## @stability stable +## @replaceable no +function checkstyle_runner +{ + declare repostatus=$1 + declare tmp=${PATCH_DIR}/$$.${RANDOM} + declare j + declare i=0 + declare fn + declare savestart=${TIMER} + declare savestop + declare output + declare logfile + declare modulesuffix + declare cmd + declare logline + declare text + declare linenum + declare codeline + + # first, let's clear out any previous run information + modules_reset + + # loop through the modules we've been given + #shellcheck disable=SC2153 + until [[ $i -eq ${#MODULE[@]} ]]; do + + # start the clock per module, setup some help vars, etc + start_clock + fn=$(module_file_fragment "${MODULE[${i}]}") + modulesuffix=$(basename "${MODULE[${i}]}") + output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" + logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" + + buildtool_cwd "${i}" + + case ${BUILDTOOL} in + ant) + cmd="${ANT} \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \ + ${ANT_ARGS[*]} checkstyle" + ;; + maven) + cmd="${MAVEN} ${MAVEN_ARGS[*]} \ + checkstyle:${CHECKSTYLE_GOAL} \ + ${CHECKSTYLE_OPTIONS} \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" + ;; + *) + UNSUPPORTED_TEST=true + return 0 + ;; + esac + + # we're going to execute it and pull out + # anything that beings with a /. that's + # almost certainly checkstyle output. + # checkstyle 6.14 or upper adds severity + # to the beginning of line, so removing it + + #shellcheck disable=SC2086 + echo_and_redirect "${logfile}" ${cmd} + ${SED} -e "s,^\[ERROR\] ,,g" -e "s,^\[WARN\] ,,g" "${logfile}" \ + | ${GREP} ^/ \ + | ${SED} -e "s,${BASEDIR},.,g" \ + > "${tmp}" + + if [[ $? == 0 ]] ; then + module_status ${i} +1 "${logfile}" "${BUILDMODEMSG} ${modulesuffix} passed checkstyle" + else + module_status ${i} -1 "${logfile}" "${BUILDMODEMSG} ${modulesuffix} failed checkstyle" + ((result = result + 1)) + fi + + # if we have some output, we need to do more work: + if [[ -s ${tmp} && "${BUILDMODE}" = patch ]]; then + + # first, let's pull out all of the files that + # we actually care about, esp since that run + # above is likely from the entire source + # this will grealy cut down how much work we + # have to do later + + for j in "${CHANGED_FILES[@]}"; do + ${GREP} "${j}" "${tmp}" >> "${tmp}.1" + done + + + # now that we have just the files we care about, + # let's unscrew it. You see... + + # checkstyle seems to do everything it possibly can + # to make it hard to process, including inconsistent + # output (sometimes it has columns, sometimes it doesn't!) + # and giving very generic errors when context would be + # helpful, esp when doing diffs. + + # in order to help calcdiff and the user out, we're + # going to reprocess the output to include the code + # line being flagged. When calcdiff gets a hold of this + # it will have the code to act as context to help + # report the correct line + + # file:linenum:(column:)error ====> + # file:linenum:code(:column)\x01:error + # \x01 will later used to identify the begining + # of the checkstyle error message + pushd "${BASEDIR}" >/dev/null + while read -r logline; do + file=$(echo "${logline}" | cut -f1 -d:) + linenum=$(echo "${logline}" | cut -f2 -d:) + text=$(echo "${logline}" | cut -f3- -d:) + codeline=$(head -n "+${linenum}" "${file}" | tail -1 ) + { + echo -n "${file}:${linenum}:${codeline}" + echo -ne "\x01" + echo ":${text}" + } >> "${output}" + done < <(cat "${tmp}.1") + + popd >/dev/null + # later on, calcdiff will turn this into code(:column):error + # compare, and then put the file:line back onto it. + else + cp -p "${tmp}" "${output}" + fi + + rm "${tmp}" "${tmp}.1" 2>/dev/null + + savestop=$(stop_clock) + #shellcheck disable=SC2034 + MODULE_STATUS_TIMER[${i}]=${savestop} + + popd >/dev/null + ((i=i+1)) + done + + TIMER=${savestart} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + checkstyle_preapply + else + checkstyle_postapply + fi +} + +function checkstyle_preapply +{ + local result + + if ! verify_needed_test checkstyle; then + return 0 + fi + + big_console_header "checkstyle: ${PATCH_BRANCH}" + + start_clock + + personality_modules branch checkstyle + checkstyle_runner branch + result=$? + modules_messages branch checkstyle true + + # keep track of how much as elapsed for us already + CHECKSTYLE_TIMER=$(stop_clock) + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postapply +{ + declare result + declare module + declare mod + declare fn + declare i=0 + declare numbranch=0 + declare numpatch=0 + declare addpatch=0 + declare summarize=true + + if ! verify_needed_test checkstyle; then + return 0 + fi + + big_console_header "checkstyle: ${BUILDMODE}" + + start_clock + + personality_modules patch checkstyle + checkstyle_runner patch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKSTYLE_TIMER}" + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + + # if there is no comparison to be done, + # we can speed this up tremendously + if [[ "${BUILDMODE}" = full ]]; then + touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" + cp -p "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/diff-checkstyle-${fn}.txt" + else + + # call calcdiffs to allow overrides + calcdiffs \ + "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ + checkstyle \ + > "${PATCH_DIR}/diff-checkstyle-${fn}.txt" + fi + + #shellcheck disable=SC2016 + numbranch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + #shellcheck disable=SC2016 + numpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + #shellcheck disable=SC2016 + addpatch=$(wc -l "${PATCH_DIR}/diff-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + + ((fixedpatch=numbranch-numpatch+addpatch)) + + statstring=$(generic_calcdiff_status "${numbranch}" "${numpatch}" "${addpatch}" ) + + mod=${module} + if [[ ${mod} = \. ]]; then + mod=root + fi + + if [[ ${addpatch} -gt 0 ]] ; then + ((result = result + 1)) + module_status ${i} -1 "diff-checkstyle-${fn}.txt" "${mod}: ${BUILDMODEMSG} ${statstring}" + elif [[ ${fixedpatch} -gt 0 ]]; then + module_status ${i} +1 "diff-checkstyle-${fn}.txt" "${mod}: ${BUILDMODEMSG} ${statstring}" + summarize=false + fi + ((i=i+1)) + done + + modules_messages patch checkstyle "${summarize}" + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} diff --git dev-support/findbugs_YETUS-471.sh dev-support/findbugs_YETUS-471.sh new file mode 100644 index 0000000..84a807b --- /dev/null +++ dev-support/findbugs_YETUS-471.sh @@ -0,0 +1,488 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FINDBUGS_HOME=${FINDBUGS_HOME:-} +FINDBUGS_WARNINGS_FAIL_PRECHECK=false +FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=false + +add_test_type findbugs + +function findbugs_usage +{ + yetus_add_option "--findbugs-home=" "Findbugs home directory (default \${FINDBUGS_HOME})" + yetus_add_option "--findbugs-strict-precheck" "If there are Findbugs warnings during precheck, fail" + yetus_add_option "--findbugs-skip-maven-source-check" "If the buildtool is maven, then skip the source check and run Findbugs for every module" +} + +function findbugs_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --findbugs-strict-precheck) + FINDBUGS_WARNINGS_FAIL_PRECHECK=true + ;; + --findbugs-skip-maven-source-check) + FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=true + ;; + esac + done +} + +## @description initialize the findbugs plug-in +## @audience private +## @stability evolving +## @replaceable no +function findbugs_initialize +{ + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install findbugs + fi +} + +function findbugs_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ + || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then + add_test findbugs + fi + fi +} + +function findbugs_precheck +{ + declare exec + declare status=0 + + if [[ -z ${FINDBUGS_HOME} ]]; then + yetus_error "FINDBUGS_HOME was not specified." + status=1 + else + for exec in findbugs \ + computeBugHistory \ + convertXmlToText \ + filterBugs \ + setBugDatabaseInfo; do + if ! verify_command "${exec}" "${FINDBUGS_HOME}/bin/${exec}"; then + status=1 + fi + done + fi + if [[ ${status} == 1 ]]; then + add_vote_table 0 findbugs "Findbugs executables are not available." + delete_test findbugs + fi +} + +## @description Dequeue maven modules that lack java sources +## @audience private +## @stability evolving +## @replaceable no +function findbugs_maven_skipper +{ + declare -i i=0 + declare skiplist=() + declare modname + + start_clock + #shellcheck disable=SC2153 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + # If there are no java source code in the module, + # skip parsing output xml file. + if [[ ! -d "${MODULE[${i}]}/src/main/java" ]]; then + skiplist=("${skiplist[@]}" "${MODULE[$i]}") + fi + ((i=i+1)) + done + + i=0 + + for modname in "${skiplist[@]}"; do + dequeue_personality_module "${modname}" + done + + if [[ -n "${modname}" ]]; then + if [[ "${BUILDMODE}" = patch ]]; then + add_vote_table 0 findbugs "Skipped patched modules with no Java source: ${skiplist[*]}" + else + add_vote_table 0 findbugs "Skipped ${#skiplist[@]} modules in the source tree with no Java source." + fi + fi +} + +## @description Run the maven findbugs plugin and record found issues in a bug database +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +## @param repostatus +function findbugs_runner +{ + local name=$1 + local module + local result=0 + local fn + local warnings_file + local i=0 + local savestop + + personality_modules "${name}" findbugs + + # strip out any modules that aren't actually java modules + # this can save a lot of time during testing + if [[ "${BUILDTOOL}" = maven && ${FINDBUGS_SKIP_MAVEN_SOURCE_CHECK} == false ]]; then + findbugs_maven_skipper + fi + + "${BUILDTOOL}_modules_worker" "${name}" findbugs + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + #shellcheck disable=SC2153 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + fn=$(module_file_fragment "${module}") + + case ${BUILDTOOL} in + maven) + file="${module}/target/findbugsXml.xml" + ;; + ant) + file="${ANT_FINDBUGSXML}" + ;; + esac + + if [[ ! -f ${file} ]]; then + module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})" + ((i=i+1)) + continue + fi + + warnings_file="${PATCH_DIR}/${name}-findbugs-${fn}-warnings" + + cp -p "${file}" "${warnings_file}.xml" + + if [[ ${name} == branch ]]; then + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${PATCH_BRANCH}" \ + "${warnings_file}.xml" "${warnings_file}.xml" + else + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name patch \ + "${warnings_file}.xml" "${warnings_file}.xml" + fi + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run setBugDatabaseInfo from findbugs" + ((result=result+1)) + ((i=i+1)) + continue + fi + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html \ + "${warnings_file}.xml" \ + "${warnings_file}.html" + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + fi + + if [[ -z ${FINDBUGS_VERSION} + && ${name} == branch ]]; then + FINDBUGS_VERSION=$(${GREP} -i "BugCollection version=" "${warnings_file}.xml" \ + | cut -f2 -d\" \ + | cut -f1 -d\" ) + if [[ -n ${FINDBUGS_VERSION} ]]; then + add_footer_table findbugs "v${FINDBUGS_VERSION}" + fi + fi + + ((i=i+1)) + done + return ${result} +} + +## @description Track pre-existing findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_preapply +{ + declare fn + declare module + declare modindex=0 + declare warnings_file + declare module_findbugs_warnings + declare result=0 + declare msg + + if ! verify_needed_test findbugs; then + return 0 + fi + + big_console_header "findbugs detection: ${PATCH_BRANCH}" + + findbugs_runner branch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + until [[ ${modindex} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${modindex}]} == -1 ]]; then + ((result=result+1)) + ((modindex=modindex+1)) + continue + fi + + module=${MODULE[${modindex}]} + start_clock + offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" + fn=$(module_file_fragment "${module}") + warnings_file="${PATCH_DIR}/branch-findbugs-${fn}-warnings" + # shellcheck disable=SC2016 + module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "${PATCH_BRANCH}" \ + "${warnings_file}.xml" \ + "${warnings_file}.xml" \ + | ${AWK} '{print $1}') + + if [[ ${module_findbugs_warnings} -gt 0 ]] ; then + msg="${module} in ${PATCH_BRANCH} has ${module_findbugs_warnings} extant Findbugs warnings." + if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" = "true" ]]; then + module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" + ((result=result+1)) + elif [[ "${BUILDMODE}" = full ]]; then + module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" + ((result=result+1)) + populate_test_table FindBugs "module:${module}" + #shellcheck disable=SC2162 + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${warnings_file}.xml") + else + module_status ${modindex} 0 "branch-findbugs-${fn}-warnings.html" "${msg}" + fi + fi + + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${modindex}]=${savestop} + ((modindex=modindex+1)) + done + modules_messages branch findbugs true + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Verify patch does not trigger any findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_postinstall +{ + declare module + declare fn + declare combined_xml + declare branchxml + declare patchxml + declare newbugsbase + declare fixedbugsbase + declare branch_warnings + declare patch_warnings + declare fixed_warnings + declare line + declare firstpart + declare secondpart + declare i=0 + declare result=0 + declare savestop + declare summarize=true + declare statstring + + if ! verify_needed_test findbugs; then + return 0 + fi + + big_console_header "findbugs detection: ${BUILDMODE}" + + findbugs_runner patch + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + + buildtool_cwd "${i}" + + fn=$(module_file_fragment "${module}") + + combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml" + branchxml="${PATCH_DIR}/branch-findbugs-${fn}-warnings.xml" + patchxml="${PATCH_DIR}/patch-findbugs-${fn}-warnings.xml" + + if [[ -f "${branchxml}" ]]; then + # shellcheck disable=SC2016 + branch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "${PATCH_BRANCH}" \ + "${branchxml}" \ + "${branchxml}" \ + | ${AWK} '{print $1}') + else + branchxml=${patchxml} + fi + + newbugsbase="${PATCH_DIR}/new-findbugs-${fn}" + fixedbugsbase="${PATCH_DIR}/fixed-findbugs-${fn}" + + "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \ + -output "${combined_xml}" \ + "${branchxml}" \ + "${patchxml}" + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + # shellcheck disable=SC2016 + patch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "patch" \ + "${patchxml}" \ + "${patchxml}" \ + | ${AWK} '{print $1}') + + #shellcheck disable=SC2016 + add_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \ + "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#1) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + #shellcheck disable=SC2016 + fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \ + "${combined_xml}" "${fixedbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + statstring=$(generic_calcdiff_status "${branch_warnings}" "${patch_warnings}" "${add_warnings}") + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \ + "${newbugsbase}.html" + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + if [[ ${add_warnings} -gt 0 ]] ; then + populate_test_table FindBugs "module:${module}" + #shellcheck disable=SC2162 + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newbugsbase}.xml") + + module_status ${i} -1 "new-findbugs-${fn}.html" "${module} ${statstring}" + ((result=result+1)) + elif [[ ${fixed_warnings} -gt 0 ]]; then + module_status ${i} +1 "" "${module} ${statstring}" + summarize=false + fi + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + popd >/dev/null + ((i=i+1)) + done + + modules_messages patch findbugs "${summarize}" + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function findbugs_rebuild +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + findbugs_preapply + else + findbugs_postinstall + fi +} diff --git dev-support/hive-personality.sh dev-support/hive-personality.sh new file mode 100644 index 0000000..b753ace --- /dev/null +++ dev-support/hive-personality.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Override these to match Apache Hadoop's requirements + +personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="http://cwiki.apache.org/confluence/display/Hive/HowToContribute" + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^HIVE-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hive" + #shellcheck disable=SC2034 + PYLINT_OPTIONS="--indent-string=' '" + #shellcheck disable=SC2034 + FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=true + #shellcheck disable=SC2034 + WHITESPACE_EOL_IGNORE_LIST='.*.q.out' + #shellcheck disable=SC2034 + WHITESPACE_TABS_IGNORE_LIST='.*.q.out' +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare extra="" + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + findbugs) + extra="-DskipTests -Pitests,findbugs" + ;; + javadoc) + extra="-DskipTests -Pitests,javadoc" + ;; + *) + extra="-DskipTests -Pitests" + ;; + esac + + # We add the changed modules with the appropriate extras + for module in "${CHANGED_MODULES[@]}"; do + if [ "${module}" == "itests" ]; then + # We do not want to test itests itself + continue; + fi + # Skip findbugs if there is no java source in the module + if [[ ${testtype} = findbugs ]]; then + if [[ ! -d "${module}/src/java" && ! -d "${module}/src/main/java" ]]; then + continue; + fi + fi + personality_enqueue_module "${module}" "${extra}" + done +} + diff --git dev-support/smart-apply-patch.sh dev-support/smart-apply-patch.sh new file mode 100644 index 0000000..a73db4d --- /dev/null +++ dev-support/smart-apply-patch.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P) +exec "${BINDIR}/yetus-wrapper.sh” smart-apply-patch --personality=${BINDIR}/hive-personality.sh "$@" diff --git dev-support/test-patch.sh dev-support/test-patch.sh new file mode 100755 index 0000000..7a532ac --- /dev/null +++ dev-support/test-patch.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P) +exec "${BINDIR}/yetus-wrapper.sh" test-patch --personality=${BINDIR}/hive-personality.sh --skip-dir=dev-support "$@" diff --git dev-support/yetus-wrapper.sh dev-support/yetus-wrapper.sh new file mode 100755 index 0000000..bc409f1 --- /dev/null +++ dev-support/yetus-wrapper.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +## @description Print a message to stderr +## @audience public +## @stability stable +## @replaceable no +## @param string +function yetus_error +{ + echo "$*" 1>&2 +} + +## @description Given a filename or dir, return the absolute version of it +## @audience public +## @stability stable +## @param directory +## @replaceable no +## @return 0 success +## @return 1 failure +## @return stdout abspath +function yetus_abs +{ + declare obj=$1 + declare dir + declare fn + + if [[ ! -e ${obj} ]]; then + return 1 + elif [[ -d ${obj} ]]; then + dir=${obj} + else + dir=$(dirname -- "${obj}") + fn=$(basename -- "${obj}") + fn="/${fn}" + fi + + dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) + if [[ $? = 0 ]]; then + echo "${dir}${fn}" + return 0 + fi + return 1 +} + + +WANTED="$1" +shift +ARGV=("$@") + +HIVE_YETUS_VERSION=${HIVE_YETUS_VERSION:-0.4.0} +BIN=$(yetus_abs "${BASH_SOURCE-$0}") +BINDIR=$(dirname "${BIN}") + +### +### if YETUS_HOME is set, then try to use it +### +if [[ -n "${YETUS_HOME}" + && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then + cp ${BINDIR}/findbugs_YETUS-471.sh ${YETUS_HOME}/lib/precommit/test-patch.d/findbugs.sh + cp ${BINDIR}/checkstyle_YETUS-484.sh ${YETUS_HOME}/lib/precommit/test-patch.d/checkstyle.sh + exec "${YETUS_HOME}/bin/${WANTED}" "${ARGV[@]}" +fi + +# +# this directory is ignored by git and maven +# +HIVE_PATCHPROCESS=${HIVE_PATCHPROCESS:-"${BINDIR}/../patchprocess"} + +if [[ ! -d "${HIVE_PATCHPROCESS}" ]]; then + mkdir -p "${HIVE_PATCHPROCESS}" +fi + +mytmpdir=$(yetus_abs "${HIVE_PATCHPROCESS}") +if [[ $? != 0 ]]; then + yetus_error "yetus-dl: Unable to cwd to ${HIVE_PATCHPROCESS}" + exit 1 +fi +HIVE_PATCHPROCESS=${mytmpdir} + +## +## if we've already DL'd it, then short cut +## +if [[ -x "${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/bin/${WANTED}" ]]; then + cp ${BINDIR}/findbugs_YETUS-471.sh ${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/lib/precommit/test-patch.d/findbugs.sh + cp ${BINDIR}/checkstyle_YETUS-484.sh ${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/lib/precommit/test-patch.d/checkstyle.sh + exec "${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}" +fi + +## +## need to DL, etc +## + +BASEURL="https://archive.apache.org/dist/yetus/${HIVE_YETUS_VERSION}/" +TARBALL="yetus-${HIVE_YETUS_VERSION}-bin.tar" + +GPGBIN=$(command -v gpg) +CURLBIN=$(command -v curl) + +pushd "${HIVE_PATCHPROCESS}" >/dev/null +if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: Cannot pushd to ${HIVE_PATCHPROCESS}" + exit 1 +fi + +if [[ -n "${CURLBIN}" ]]; then + "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz" + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz" + exit 1 + fi +else + yetus_error "ERROR: yetus-dl requires curl." + exit 1 +fi + +if [[ -n "${GPGBIN}" ]]; then + mkdir -p .gpg + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: Unable to create ${HIVE_PATCHPROCESS}/.gpg" + exit 1 + fi + chmod -R 700 .gpg + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: Unable to chmod ${HIVE_PATCHPROCESS}/.gpg" + exit 1 + fi + "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS" + exit 1 + fi + "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc" + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc" + exit 1 + fi + "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --import "${HIVE_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1 + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: gpg unable to import ${HIVE_PATCHPROCESS}/KEYS_YETUS" + exit 1 + fi + "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1 + if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HIVE_PATCHPROCESS} failed" + exit 1 + fi +fi + +gunzip -c "${TARBALL}.gz" | tar xpf - +if [[ $? != 0 ]]; then + yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HIVE_PATCHPROCESS} to try again." + exit 1 +fi + +if [[ -x "${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/bin/${WANTED}" ]]; then + popd >/dev/null + cp ${BINDIR}/findbugs_YETUS-471.sh ${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/lib/precommit/test-patch.d/findbugs.sh + cp ${BINDIR}/checkstyle_YETUS-484.sh ${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/lib/precommit/test-patch.d/checkstyle.sh + exec "${HIVE_PATCHPROCESS}/yetus-${HIVE_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}" +fi + +## +## give up +## +yetus_error "ERROR: ${WANTED} is not part of Apache Yetus ${HIVE_YETUS_VERSION}" +exit 1 diff --git pom.xml pom.xml index 3047698..d5ab483 100644 --- pom.xml +++ pom.xml @@ -1334,6 +1334,21 @@ findbugs + + + + org.codehaus.mojo + findbugs-maven-plugin + 3.0.0 + + true + 2048 + -Djava.awt.headless=true -Xmx2048m -Xms512m + ${basedir}/${hive.path.to.root}/findbugs/findbugs-exclude.xml + + + +