diff --git dev-support/flaky-test-detector.sh dev-support/flaky-test-detector.sh new file mode 100755 index 0000000..3d9dd28 --- /dev/null +++ dev-support/flaky-test-detector.sh @@ -0,0 +1,61 @@ +#!/bin/bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +# A tool which pulls test case results from Jenkins server. It displays a union of failed +# test cases from the last 15(by default and actual number of jobs can be less depending +# on availablity) runs recorded in Jenkins sever and track how each of them are performed for +# all the last 15 runs(passed, not run or failed) +# +# You can see more details ./jenkins-tools/README.md +# +# flaky-test-detector.sh is a wrapper script of buildstats.jar. It downloads buildstats.jar from +# http://people.apache.org/~enis/buildstats.jar to avoid building the jar inside hbase build enviroment. +# buildstats.jar can be built by following instructions ./jenkins-tools/README.md +# +# Usage: ./flaky-test-detector.sh [number of last most recent jobs to check] +# +# Sample command: +# ./flaky-test-detector.sh https://builds.apache.org HBase-TRUNK 15 +# +## + +if [ $# -lt 2 ] +then +echo "Usage: ./flaky-test-detector.sh [number of last most recent jobs to check]" +exit +fi + +NUM_BUILDS=15 +JAR_URL=http://people.apache.org/~enis/buildstats.jar +JAR_LOCATION=/tmp/buildstats.jar +WGET=${WGET:-wget} +if [ $# -eq 3 ] +then +NUM_BUILDS=$3 +fi + +if [ ! -f $JAR_LOCATION ]; +then +echo "Downloading buildstats.jar from "$JAR_URL +$WGET -q -O $JAR_LOCATION $JAR_URL +fi + +echo "Fetching test results of build job "$2" from "$1 +echo "" +java -jar $JAR_LOCATION $1 $2 $NUM_BUILDS \ No newline at end of file diff --git dev-support/jenkins-tools/README.md dev-support/jenkins-tools/README.md new file mode 100644 index 0000000..d16c974 --- /dev/null +++ dev-support/jenkins-tools/README.md @@ -0,0 +1,49 @@ +jenkins-tools +============= + +A tool which pulls test case results from Jenkins server. It displays a union of failed test cases +from the last 15(by default and actual number of jobs can be less depending on availablity) runs +recorded in Jenkins sever and track how each of them are performed for all the last 15 runs(passed, +not run or failed) + +Pre-requirement(run under folder jenkins-tools) + Please download jenkins-client from https://github.com/cosmin/jenkins-client + 1) git clone git://github.com/cosmin/jenkins-client.git + 2) make sure the dependency jenkins-client version in ./buildstats/pom.xml matches the + downloaded jenkins-client(current value is 0.1.4-SNAPSHOT) + +Build command(run under folder jenkins-tools): + + mvn package + +Usage are: + + java -jar ./buildstats/target/buildstats.jar [number of last most recent jobs to check] + +Sample commands are: + + java -jar ./buildstats/target/buildstats.jar https://builds.apache.org HBase-TRUNK + +Sample output(where 1 means "PASSED", 0 means "NOT RUN AT ALL", -1 means "FAILED"): + +Failed Test Cases 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3632 3633 3634 3635 + +org.apache.hadoop.hbase.catalog.testmetareadereditor.testretrying 1 1 -1 0 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.client.testadmin.testdeleteeditunknowncolumnfamilyandortable 0 1 1 1 -1 0 1 1 0 1 1 1 1 1 +org.apache.hadoop.hbase.client.testfromclientsidewithcoprocessor.testclientpoolthreadlocal 1 1 1 1 1 1 1 1 0 1 1 -1 0 1 +org.apache.hadoop.hbase.client.testhcm.testregioncaching 1 1 -1 0 1 1 -1 0 -1 0 -1 0 1 1 +org.apache.hadoop.hbase.client.testmultiparallel.testflushcommitswithabort 1 1 1 1 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.client.testscannertimeout.test3686a 1 1 1 1 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.coprocessor.example.testrowcountendpoint.org.apache.hadoop.hbase.coprocessor.example.testrowcountendpoint 0 -1 0 -1 0 0 0 -1 0 0 0 0 0 0 +org.apache.hadoop.hbase.coprocessor.example.testzookeeperscanpolicyobserver.org.apache.hadoop.hbase.coprocessor.example.testzookeeperscanpolicyobserver 0 -1 0 -1 0 0 0 -1 0 0 0 0 0 0 +org.apache.hadoop.hbase.master.testrollingrestart.testbasicrollingrestart 1 1 1 1 -1 0 1 1 1 1 1 1 -1 0 +org.apache.hadoop.hbase.regionserver.testcompactionstate.testmajorcompaction 1 1 -1 0 1 1 1 1 1 1 1 1 1 1 +org.apache.hadoop.hbase.regionserver.testcompactionstate.testminorcompaction 1 1 -1 0 1 1 1 1 1 1 1 1 1 1 +org.apache.hadoop.hbase.replication.testreplication.loadtesting 1 1 1 1 1 1 1 1 1 -1 0 1 1 1 +org.apache.hadoop.hbase.rest.client.testremoteadmin.org.apache.hadoop.hbase.rest.client.testremoteadmin 0 0 0 0 0 0 0 0 -1 0 0 0 0 0 +org.apache.hadoop.hbase.rest.client.testremotetable.org.apache.hadoop.hbase.rest.client.testremotetable 0 0 0 0 0 0 0 0 -1 0 0 0 0 0 +org.apache.hadoop.hbase.security.access.testtablepermissions.testbasicwrite 0 1 1 1 1 1 1 1 1 1 1 1 1 -1 +org.apache.hadoop.hbase.testdrainingserver.testdrainingserverwithabort 1 1 1 1 1 -1 0 1 1 1 1 1 -1 0 +org.apache.hadoop.hbase.util.testhbasefsck.testregionshouldnotbedeployed 1 1 1 1 1 1 -1 0 -1 0 -1 -1 0 -1 + + diff --git dev-support/jenkins-tools/buildstats/pom.xml dev-support/jenkins-tools/buildstats/pom.xml new file mode 100644 index 0000000..966a809 --- /dev/null +++ dev-support/jenkins-tools/buildstats/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + buildstats + buildstats + 1.0 + jar + + buildstats + http://maven.apache.org + + + UTF-8 + + + + + com.offbytwo.jenkins + jenkins-client + 0.1.4-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + true + + + + true + true + + + + + + + maven-assembly-plugin + + + + true + org.apache.hadoop.hbase.devtools.buildstats.TestResultHistory + + + + jar-with-dependencies + + buildstats + false + + + + make-my-jar-with-dependencies + package + + single + + + + + + + + diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java new file mode 100644 index 0000000..d41af0b --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.model.BaseModel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class BuildResultWithTestCaseDetails extends BaseModel { + + List suites; + + /* default constructor needed for Jackson */ + public BuildResultWithTestCaseDetails() { + this(new ArrayList()); + } + + public BuildResultWithTestCaseDetails(List s) { + this.suites = s; + } + + public BuildResultWithTestCaseDetails(TestSuite... s) { + this(Arrays.asList(s)); + } + + public List getSuites() { + return suites; + } + + public void setSuites(List s) { + suites = s; + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java new file mode 100644 index 0000000..1a79fff --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class HistoryReport { + private List buildsWithTestResults; + private Map historyResults; + + public HistoryReport() { + buildsWithTestResults = new ArrayList(); + this.historyResults = new HashMap(); + } + + public Map getHistoryResults() { + return this.historyResults; + } + + public List getBuildsWithTestResults() { + return this.buildsWithTestResults; + } + + public void setBuildsWithTestResults(List src) { + this.buildsWithTestResults = src; + } + + public void setHistoryResults(Map src) { + this.historyResults = src; + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java new file mode 100644 index 0000000..2179aff --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +public class TestCaseResult { + private String className; + private int failedSince; + private String name; + private String status; + + public String getName() { + return name; + } + + public String getClassName() { + return className; + } + + public int failedSince() { + return failedSince; + } + + public String getStatus() { + return status; + } + + public void setName(String s) { + name = s; + } + + public void setClassName(String s) { + className = s; + } + + public void setFailedSince(int s) { + failedSince = s; + } + + public void setStatus(String s) { + status = s; + } + + public String getFullName() { + return (this.className + "." + this.name).toLowerCase(); + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java new file mode 100644 index 0000000..94b1a90 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.JenkinsServer; +import com.offbytwo.jenkins.client.JenkinsHttpClient; +import com.offbytwo.jenkins.model.*; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + + +public class TestResultHistory +{ + public final static String STATUS_REGRESSION = "REGRESSION"; + public final static String STATUS_FAILED = "FAILED"; + public final static String STATUS_PASSED = "PASSED"; + public final static String STATUS_FIXED = "FIXED"; + public static int BUILD_HISTORY_NUM = 15; + + private JenkinsHttpClient client; + private String jobName; + + public TestResultHistory(String apacheHTTPURL, String jobName, String userName, String passWord) + throws URISyntaxException { + this.client = new JenkinsHttpClient(new URI(apacheHTTPURL), userName, passWord); + this.jobName = jobName; + } + + public static void main(String[] args) { + + if(args.length < 2) { + printUsage(); + return; + } + + String apacheHTTPUrl = args[0]; + String jobName = args[1]; + if(args.length > 2) { + int tmpHistoryJobNum = -1; + try { + tmpHistoryJobNum = Integer.parseInt(args[2]); + } catch (NumberFormatException ex) { + // ignore + } + if(tmpHistoryJobNum > 0) { + BUILD_HISTORY_NUM = tmpHistoryJobNum; + } + } + + try { + TestResultHistory buildHistory = new TestResultHistory(apacheHTTPUrl, jobName, "", ""); + HistoryReport report = buildHistory.getReport(); + + // display result in console + System.out.printf("%-30s", "Failed Test Cases"); + for(Integer i : report.getBuildsWithTestResults()) { + System.out.printf("%5d", i); + } + System.out.println("\n========================================================"); + SortedSet keys = new TreeSet(report.getHistoryResults().keySet()); + for (String failedTestCase : keys) { + System.out.println(); + int[] resultHistory = report.getHistoryResults().get(failedTestCase); + System.out.print(failedTestCase); + for(int i = 0 ; i < resultHistory.length ; i++) { + System.out.printf("%5d", resultHistory[i]); + } + } + System.out.println(); + } catch (Exception ex) { + System.out.println("Got unexpected exception: " + ex.getMessage()); + } + } + + protected static void printUsage() { + System.out.println(" [Number of Historical Jobs to Check]"); + System.out.println("Sample Input: \"https://builds.apache.org\" " + + "\"HBase-TRUNK-on-Hadoop-2.0.0\" "); + } + + public HistoryReport getReport() { + HistoryReport report = new HistoryReport(); + + List buildWithTestResults = new ArrayList(); + Map failureStats = new HashMap(); + + try { + JenkinsServer jenkins = new JenkinsServer(this.client); + Map jobs = jenkins.getJobs(); + JobWithDetails job = jobs.get(jobName.toLowerCase()).details(); + + // build test case failures stats for the past 10 builds + Build lastBuild = job.getLastBuild(); + int startingBuildNumber = (lastBuild.getNumber() - BUILD_HISTORY_NUM > 0) ? + lastBuild.getNumber() - BUILD_HISTORY_NUM + 1: + 1; + + Map> executedTestCases = new HashMap>(); + + String buildUrl = lastBuild.getUrl(); + for(int i = startingBuildNumber; i <= lastBuild.getNumber(); i++) { + HashMap buildExecutedTestCases = new HashMap(2048); + String curBuildUrl = buildUrl.replaceFirst("/" + lastBuild.getNumber(), "/" + i); + List failedCases = null; + try { + failedCases = getBuildFailedTestCases(curBuildUrl, buildExecutedTestCases); + buildWithTestResults.add(i); + } catch (Exception ex) { + // can't get result so skip it + continue; + } + executedTestCases.put(i, buildExecutedTestCases); + + // set test result failed cases of current build + for(String curFailedTestCase : failedCases) { + if(failureStats.containsKey(curFailedTestCase)) { + int[] testCaseResultArray = failureStats.get(curFailedTestCase); + testCaseResultArray[i - startingBuildNumber] = -1; + } else { + int[] testResult = new int[BUILD_HISTORY_NUM]; + testResult[i - startingBuildNumber] = -1; + // refill previous build test results for newly failed test case + for(int k = startingBuildNumber; k < i ; k++) { + HashMap tmpBuildExecutedTestCases = executedTestCases.get(k); + if(tmpBuildExecutedTestCases != null + && tmpBuildExecutedTestCases.containsKey(curFailedTestCase)) { + String statusStr = tmpBuildExecutedTestCases.get(curFailedTestCase); + testResult[k-startingBuildNumber] = convertStatusStringToInt(statusStr); + } + } + failureStats.put(curFailedTestCase, testResult); + } + + } + + // set test result for previous failed test cases + for(String curTestCase : failureStats.keySet()) { + if(!failedCases.contains(curTestCase) && buildExecutedTestCases.containsKey(curTestCase)) { + String statusVal = buildExecutedTestCases.get(curTestCase); + int[] testCaseResultArray = failureStats.get(curTestCase); + testCaseResultArray[i - startingBuildNumber] = convertStatusStringToInt(statusVal); + } + } + } + + report.setBuildsWithTestResults(buildWithTestResults); + for (String failedTestCase : failureStats.keySet()) { + int[] resultHistory = failureStats.get(failedTestCase); + int[] compactHistory = new int[buildWithTestResults.size()]; + int index = 0; + for(Integer i : buildWithTestResults) { + compactHistory[index] = resultHistory[i-startingBuildNumber]; + index++; + } + failureStats.put(failedTestCase, compactHistory); + } + + report.setHistoryResults(failureStats); + + } catch(Exception ex) { + System.out.println(ex); + } + + return report; + } + + /** + * + * @param statusVal + * @return 1 means PASSED, -1 means FAILED, 0 means SKIPPED + */ + static int convertStatusStringToInt(String statusVal) { + + if(statusVal.equalsIgnoreCase(STATUS_REGRESSION) + || statusVal.equalsIgnoreCase(STATUS_FAILED)) { + return -1; + } else if (statusVal.equalsIgnoreCase(STATUS_PASSED)){ + return 1; + } + + return 0; + } + + /** + * Get failed test cases of a build + * + * @param buildURL Jenkins build job URL + * @param executedTestCases Set of test cases which was executed for the build + * @return list of failed test case names + */ + List getBuildFailedTestCases(String buildURL, HashMap executedTestCases) + throws IOException { + List result = new ArrayList(); + + String apiPath = urlJoin(buildURL, + "testReport?depth=10&tree=suites[cases[className,name,status,failedSince]]"); + + List suites = + client.get(apiPath, BuildResultWithTestCaseDetails.class).getSuites(); + + result = getTestSuiteFailedTestcase(suites, executedTestCases); + + return result; + } + + private List getTestSuiteFailedTestcase(List suites, HashMap executedTestCases) { + List result = new ArrayList(); + + if(suites == null) { + return result; + } + + for(TestSuite curTestSuite: suites) { + for(TestCaseResult curTestCaseResult: curTestSuite.getCases()) { + if(curTestCaseResult.getStatus().equalsIgnoreCase(STATUS_FAILED) || + curTestCaseResult.getStatus().equalsIgnoreCase(STATUS_REGRESSION)) { + // failed test case + result.add(curTestCaseResult.getFullName()); + } + executedTestCases.put(curTestCaseResult.getFullName(), curTestCaseResult.getStatus()); + } + } + + return result; + } + + String urlJoin(String path1, String path2) { + if (!path1.endsWith("/")) { + path1 += "/"; + } + if (path2.startsWith("/")) { + path2 = path2.substring(1); + } + return path1 + path2; + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java new file mode 100644 index 0000000..97531eb --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.model.BaseModel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class TestSuite extends BaseModel{ + List cases; + + public TestSuite() { + this(new ArrayList()); + } + + public TestSuite(List s) { + this.cases = s; + } + + public TestSuite(TestCaseResult... s) { + this(Arrays.asList(s)); + } + + public List getCases() { + return cases; + } + + public void setCases(List s) { + cases = s; + } +} diff --git dev-support/jenkins-tools/pom.xml dev-support/jenkins-tools/pom.xml new file mode 100644 index 0000000..a276f79 --- /dev/null +++ dev-support/jenkins-tools/pom.xml @@ -0,0 +1,16 @@ + + + 4.0.0 + + org.apache.hbase + jenkins-tools + 1.0 + pom + + + jenkins-client + buildstats + +