From c2e82de5c8dbb55e279f50481ce04176b12bb0c5 Mon Sep 17 00:00:00 2001 From: Apekshit Date: Wed, 13 Apr 2016 23:02:17 -0700 Subject: [PATCH] HBASE-15651 Script to report flaky tests. (Apekshit) Change-Id: I5cd5c23985b8c3f928d7ab44e57606b0a5478f15 --- dev-support/report-flakies.py | 149 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100755 dev-support/report-flakies.py diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py new file mode 100755 index 0000000..22befbf --- /dev/null +++ b/dev-support/report-flakies.py @@ -0,0 +1,149 @@ +#!/usr/bin/python +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script uses Jenkins REST api to collect test results of given builds and generates flakyness +# data about unittests. +# Print help: ./report-flakies.py -h +import json +import os +import re +import sets +import sys +import urllib2 + +def print_usage(): + print "Usage: " + sys.argv[0] + " [options] urls" + print "" + print "urls Space separated list of urls (single/multi-configuration project) to analyze" + print "" + print "Options:" + print "-h Prints this help message" + print "-r Number of runs to analyze for each job (if available in jenkins)" + print "-mvn Writes two strings for including/excluding these flaky tests using maven" + print " flags. These strings are written to files so they can be saved as artifacts" + print " and easily imported in other projects." + +# If no urls given, print help message and quit. +if len(sys.argv) == 1: + print "urls missing.\n" + print_usage() + sys.exit(1) + +# Print help and quit. +if sys.argv[1] == "-h" or sys.argv[1] == "--help": + print_usage() + sys.exit(0) + +# Parse options in the command line. +max_runs = 10000 +write_mvn_include_exclude_strings = False +for option in sys.argv: + if option[0:3] == '-r=': + max_runs = int(option[3:]) + if option == '-mvn': + write_mvn_include_exclude_strings = True + +# Given url of an executed build, fetches its test report, and returns dictionary from testname to +# pass/skip/fail status. +def get_build_results (build_url): + print "Getting test results for " + build_url + url = build_url + "testReport/api/json?tree=suites[cases[className,name,status]]" + try: + json_response = json.loads(urllib2.urlopen(url).read()) + except urllib2.HTTPError as e: + if e.code == 404: + print "No test results for " + build_url + else: + print e + return {} + + tests = {} + for test_cases in json_response["suites"]: + for test in test_cases["cases"]: + # Truncate initial "org.apache.hadoop.hbase." from all tests. + test_name = (test["className"] + "#" + test["name"])[24:] + tests[test_name] = test["status"] + return tests + +# If any urls is of multi-configuration project, get urls for individual jobs. +jobs_list = [] +for url in sys.argv[-1].split(): + info = json.loads(urllib2.urlopen(url + "/api/json").read()) + if info.has_key("activeConfigurations"): + for config in info["activeConfigurations"]: + jobs_list.append(config["url"]) + elif info.has_key("builds"): + jobs_list.append(url) + else: + print "Error: Bad url - " + url + sys.exit(1) + +global_bad_tests = set() +# Iterates over each job, gets its test results and prints flaky tests. +for job_url in jobs_list: + print "Analyzing job: " + job_url + run_id_to_results = {} + runs = json.loads(urllib2.urlopen(job_url + "/api/json").read())["builds"] + num_runs = 0 + for run in runs: + run_id_to_results[run["number"]] = get_build_results(run["url"]) + num_runs += 1 + if num_runs == max_runs: + break + + # Collect list of bad tests. + bad_tests = set() + for run_id in run_id_to_results: + for test in run_id_to_results[run_id]: + if run_id_to_results[run_id][test] == "REGRESSION": + bad_tests.add(test) + global_bad_tests.add(test) + + # Get total and failed run times for each bad test. + run_counts = {key:dict([('total', 0), ('failed', 0)]) for key in bad_tests} + for run_id in run_id_to_results: + run_results = run_id_to_results[run_id] + for bad_test in bad_tests: + if run_results.has_key(bad_test): + if run_results[bad_test] != "SKIPPED": # Ignore the test if it's skipped. + run_counts[bad_test]['total'] += 1 + if run_results[bad_test] == "REGRESSION": + run_counts[bad_test]['failed'] += 1 + + print "{:>100} {:6} {:10} {}".format("Test Name", "Failed", "Total Runs", "Flakyness") + for bad_test in bad_tests: + fail = run_counts[bad_test]['failed'] + total = run_counts[bad_test]['total'] + print "{:>100} {:10} {:6} {:2.0f}%".format(bad_test, fail, total, fail*100.0/total) + +print global_bad_tests + +if write_mvn_include_exclude_strings: + includes = "" + excludes = "" + for test in global_bad_tests: + test = re.sub(".*\.", "", test) # Remove package name prefix. + test = re.sub("#.*", "", test) # Remove individual unittest's name + includes += test + "," + excludes += "**/" + test + ".java," + inc_file = open("./includes", "w") + inc_file.write(includes) + inc_file.close() + exc_file = open("./excludes", "w") + exc_file.write(excludes) + exc_file.close() -- 2.3.2 (Apple Git-55)