From 67b79764868b3c406632e553f667275a7a3d8656 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 30 Jul 2018 12:36:54 -0500 Subject: [PATCH 1/3] HBASE-20979 Flaky test reporting should specify what JSON it needs and handle HTTP errors --- dev-support/report-flakies.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py index 201980d35f..1b3161af6d 100755 --- a/dev-support/report-flakies.py +++ b/dev-support/report-flakies.py @@ -76,7 +76,10 @@ def get_bad_tests(build_url, is_yetus): Returns None if can't get maven output from the build or if there is any other error. """ logger.info("Analyzing %s", build_url) - response = requests.get(build_url + "/api/json").json() + needed_fields="_class,building" + if is_yetus: + needed_fields+=",artifacts[fileName,relativePath]" + response = requests.get(build_url + "/api/json?tree=" + needed_fields).json() if response["building"]: logger.info("Skipping this build since it is in progress.") return {} @@ -125,7 +128,11 @@ def expand_multi_config_projects(cli_args): excluded_builds = [] if excluded_builds_arg is not None and excluded_builds_arg[i] != "None": excluded_builds = [int(x) for x in excluded_builds_arg[i].split(",")] - response = requests.get(job_url + "/api/json").json() + request = requests.get(job_url + "/api/json?tree=_class,activeConfigurations%5Burl%5D") + if request.status_code != 200: + raise Exception("Failed to get job information from jenkins for url '" + job_url + + "'. Jenkins returned HTTP status " + str(request.status_code)) + response = request.json() if response.has_key("activeConfigurations"): for config in response["activeConfigurations"]: final_expanded_urls.append({'url':config["url"], 'max_builds': max_builds, @@ -152,7 +159,7 @@ expanded_urls = expand_multi_config_projects(args) for url_max_build in expanded_urls: url = url_max_build["url"] excludes = url_max_build["excludes"] - json_response = requests.get(url + "/api/json").json() + json_response = requests.get(url + "/api/json?tree=id,builds%5Bnumber,url%5D").json() if json_response.has_key("builds"): builds = json_response["builds"] logger.info("Analyzing job: %s", url) -- 2.16.1 From 736b43a45be6a60b2566e284888635b9a3c23c33 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Fri, 10 Aug 2018 11:28:10 -0500 Subject: [PATCH 2/3] HBASE-20387 turn flaky test tracking infra into per-branch pipeline. * gather up all the flaky test stuff into a directory * create Jenkins Pipeline DSL for the report generation and the flaky re-testing * have the nightly per-branch job consume the results of flaky reporting --- dev-support/Jenkinsfile | 6 +- dev-support/{ => flaky-tests}/findHangingTests.py | 0 .../flaky-dashboard-template.html | 0 .../flaky-tests/flaky-reporting.Jenkinsfile | 66 ++++++++++++++++++++++ dev-support/{ => flaky-tests}/report-flakies.py | 0 .../flaky-tests/run-flaky-tests.Jenkinsfile | 65 +++++++++++++++++++++ dev-support/hbase_nightly_yetus.sh | 4 -- 7 files changed, 132 insertions(+), 9 deletions(-) rename dev-support/{ => flaky-tests}/findHangingTests.py (100%) rename dev-support/{ => flaky-tests}/flaky-dashboard-template.html (100%) create mode 100644 dev-support/flaky-tests/flaky-reporting.Jenkinsfile rename dev-support/{ => flaky-tests}/report-flakies.py (100%) create mode 100644 dev-support/flaky-tests/run-flaky-tests.Jenkinsfile diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 59d3227f58..7c3538a62a 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -47,11 +47,7 @@ pipeline { ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump' // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' - // Flaky urls for different branches. Replace '-' and '.' in branch name by '_' because those - // characters are not allowed in bash variable name. - // Not excluding flakies from the nightly build for now. - // EXCLUDE_TESTS_URL_master = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/' - // EXCLUDE_TESTS_URL_branch_2 = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests-branch2.0/lastSuccessfulBuild/artifact/excludes/' + EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests-per-branch/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. diff --git a/dev-support/findHangingTests.py b/dev-support/flaky-tests/findHangingTests.py similarity index 100% rename from dev-support/findHangingTests.py rename to dev-support/flaky-tests/findHangingTests.py diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-tests/flaky-dashboard-template.html similarity index 100% rename from dev-support/flaky-dashboard-template.html rename to dev-support/flaky-tests/flaky-dashboard-template.html diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile new file mode 100644 index 0000000000..0d20336732 --- /dev/null +++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent { + node { + label 'Hadoop' + } + } + triggers { + cron('@daily') + } + options { + buildDiscarder(logRotator(numToKeepStr: '100')) + timeout (time: 15, unit: 'MINUTES') + timestamps() + } + parameters { + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + } + stages { + stage ('build flaky report') { + steps { + sh '''#!/usr/bin/env bash + set -e + if [ "${DEBUG}" = "true" ]; then + set -x + fi + declare -a flaky_args + flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5) + flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests-per-branch/job/${BRANCH_NAME}" --is-yetus False --max-builds 40) + docker build -t hbase-dev-support dev-support + docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}" +''' + } + } + } + post { + always { + // Has to be relative to WORKSPACE. + archive "includes,excludes,dashboard.html" + publishHTML target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: ".", + reportFiles: 'dashboard.html', + reportName: 'Flaky Test Report' + ] + } + } +} diff --git a/dev-support/report-flakies.py b/dev-support/flaky-tests/report-flakies.py similarity index 100% rename from dev-support/report-flakies.py rename to dev-support/flaky-tests/report-flakies.py diff --git a/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile b/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile new file mode 100644 index 0000000000..cacb1753bd --- /dev/null +++ b/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent { + node { + label 'Hadoop' + } + } + triggers { + cron('@hourly') + } + options { + // this should roughly match how long we tell the flaky dashboard to look at + buildDiscarder(logRotator(numToKeepStr: '80')) + timeout (time: 2, unit: 'HOURS') + timestamps() + } + parameters { + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + } + tools { + // this should match what the yetus nightly job for the branch will use + maven 'Maven (latest)' + jdk "JDK 1.8 (latest)" + } + stages { + stage ('run flaky tests') { + steps { + sh '''#!/usr/bin/env bash + set -e + if [ "${DEBUG}" = "true" ]; then + curl_debug="-v" + mvn_debug="-X" + set -x + fi + ulimit -a + rm -rf local-repository/org/apache/hbase + curl --fail "${curl_debug}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests-per-branch/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes" + mvn clean package "${mvn_debug}" --batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dmaven.repo.local="${WORKSPACE}/local-repository" -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3 +''' + } + } + } + post { + always { + junit testResults: "**/surefire-reports/*.xml", allowEmptyResults: true + // TODO compress these logs + archive 'includes.txt,**/surefire-reports/*,**/test-data/*' + } + } +} diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index bba5f4dc97..185a288dea 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -71,10 +71,6 @@ YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}") YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}") -# Currently, flaky list is calculated only for master branch. -UNDERSCORED_BRANCH_NAME=$(echo ${BRANCH_NAME} | tr '.-' '_') -EXCLUDE_TESTS_URL=$(eval echo "\$EXCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}") -INCLUDE_TESTS_URL=$(eval echo "\$INCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}") if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") fi -- 2.16.1 From 1d27f2f1388ac7fb1f134f35217fd31e9c80c538 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 14 Aug 2018 12:48:39 -0500 Subject: [PATCH 3/3] HBASE-20387 fixup to remove '-per-branch' from job names --- dev-support/Jenkinsfile | 2 +- dev-support/flaky-tests/flaky-reporting.Jenkinsfile | 2 +- dev-support/flaky-tests/run-flaky-tests.Jenkinsfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 7c3538a62a..bbff87cc0c 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -47,7 +47,7 @@ pipeline { ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump' // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' - EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests-per-branch/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes" + EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile index 0d20336732..dfe5b66c90 100644 --- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile +++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile @@ -41,7 +41,7 @@ pipeline { fi declare -a flaky_args flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5) - flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests-per-branch/job/${BRANCH_NAME}" --is-yetus False --max-builds 40) + flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 40) docker build -t hbase-dev-support dev-support docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}" ''' diff --git a/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile b/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile index cacb1753bd..6d361a6bb2 100644 --- a/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile +++ b/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile @@ -49,7 +49,7 @@ pipeline { fi ulimit -a rm -rf local-repository/org/apache/hbase - curl --fail "${curl_debug}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests-per-branch/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes" + curl --fail "${curl_debug}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes" mvn clean package "${mvn_debug}" --batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dmaven.repo.local="${WORKSPACE}/local-repository" -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3 ''' } -- 2.16.1