diff --git dev-support/jenkins-tools/README.md dev-support/jenkins-tools/README.md new file mode 100644 index 0000000..9e1905f --- /dev/null +++ dev-support/jenkins-tools/README.md @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jenkins-tools +============= + +A tool which pulls test case results from Jenkins server. It displays a union of failed test cases +from the last 15(by default and actual number of jobs can be less depending on availablity) runs +recorded in Jenkins sever and track how each of them are performed for all the last 15 runs(passed, +not run or failed) + +Pre-requirement(run under folder jenkins-tools) + Please download jenkins-client from https://github.com/cosmin/jenkins-client + 1) git clone git://github.com/cosmin/jenkins-client.git + 2) make sure the dependency jenkins-client version in ./buildstats/pom.xml matches the + downloaded jenkins-client(current value is 0.1.6-SNAPSHOT) + +Build command(run under folder jenkins-tools): + + mvn clean package + +Usage are: + + java -jar ./buildstats/target/buildstats.jar [number of last most recent jobs to check] + +Sample commands are: + + java -jar ./buildstats/target/buildstats.jar https://builds.apache.org HBase-TRUNK + +Sample output(where 1 means "PASSED", 0 means "NOT RUN AT ALL", -1 means "FAILED"): + +Failed Test Cases 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3632 3633 3634 3635 + +org.apache.hadoop.hbase.catalog.testmetareadereditor.testretrying 1 1 -1 0 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.client.testadmin.testdeleteeditunknowncolumnfamilyandortable 0 1 1 1 -1 0 1 1 0 1 1 1 1 1 +org.apache.hadoop.hbase.client.testfromclientsidewithcoprocessor.testclientpoolthreadlocal 1 1 1 1 1 1 1 1 0 1 1 -1 0 1 +org.apache.hadoop.hbase.client.testhcm.testregioncaching 1 1 -1 0 1 1 -1 0 -1 0 -1 0 1 1 +org.apache.hadoop.hbase.client.testmultiparallel.testflushcommitswithabort 1 1 1 1 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.client.testscannertimeout.test3686a 1 1 1 1 1 1 1 1 -1 0 1 1 1 1 +org.apache.hadoop.hbase.coprocessor.example.testrowcountendpoint.org.apache.hadoop.hbase.coprocessor.example.testrowcountendpoint 0 -1 0 -1 0 0 0 -1 0 0 0 0 0 0 +org.apache.hadoop.hbase.coprocessor.example.testzookeeperscanpolicyobserver.org.apache.hadoop.hbase.coprocessor.example.testzookeeperscanpolicyobserver 0 -1 0 -1 0 0 0 -1 0 0 0 0 0 0 +org.apache.hadoop.hbase.master.testrollingrestart.testbasicrollingrestart 1 1 1 1 -1 0 1 1 1 1 1 1 -1 0 +org.apache.hadoop.hbase.regionserver.testcompactionstate.testmajorcompaction 1 1 -1 0 1 1 1 1 1 1 1 1 1 1 +org.apache.hadoop.hbase.regionserver.testcompactionstate.testminorcompaction 1 1 -1 0 1 1 1 1 1 1 1 1 1 1 +org.apache.hadoop.hbase.replication.testreplication.loadtesting 1 1 1 1 1 1 1 1 1 -1 0 1 1 1 +org.apache.hadoop.hbase.rest.client.testremoteadmin.org.apache.hadoop.hbase.rest.client.testremoteadmin 0 0 0 0 0 0 0 0 -1 0 0 0 0 0 +org.apache.hadoop.hbase.rest.client.testremotetable.org.apache.hadoop.hbase.rest.client.testremotetable 0 0 0 0 0 0 0 0 -1 0 0 0 0 0 +org.apache.hadoop.hbase.security.access.testtablepermissions.testbasicwrite 0 1 1 1 1 1 1 1 1 1 1 1 1 -1 +org.apache.hadoop.hbase.testdrainingserver.testdrainingserverwithabort 1 1 1 1 1 -1 0 1 1 1 1 1 -1 0 +org.apache.hadoop.hbase.util.testhbasefsck.testregionshouldnotbedeployed 1 1 1 1 1 1 -1 0 -1 0 -1 -1 0 -1 + + diff --git dev-support/jenkins-tools/buildstats/pom.xml dev-support/jenkins-tools/buildstats/pom.xml new file mode 100644 index 0000000..4149dc7 --- /dev/null +++ dev-support/jenkins-tools/buildstats/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + org.apache.hbase + buildstats + 1.0 + jar + buildstats + + + UTF-8 + + + + + com.offbytwo.jenkins + jenkins-client + 0.1.6-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + true + + + + true + true + + + + + + + maven-assembly-plugin + + + + true + org.apache.hadoop.hbase.devtools.buildstats.TestResultHistory + + + + jar-with-dependencies + + buildstats + false + + + + make-my-jar-with-dependencies + package + + single + + + + + + + + diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java new file mode 100644 index 0000000..ad3f0e3 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/BuildResultWithTestCaseDetails.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.model.BaseModel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class BuildResultWithTestCaseDetails extends BaseModel { + + List suites; + + /* default constructor needed for Jackson */ + public BuildResultWithTestCaseDetails() { + this(new ArrayList()); + } + + public BuildResultWithTestCaseDetails(List s) { + this.suites = s; + } + + public BuildResultWithTestCaseDetails(TestSuite... s) { + this(Arrays.asList(s)); + } + + public List getSuites() { + return suites; + } + + public void setSuites(List s) { + suites = s; + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java new file mode 100644 index 0000000..80671b2 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/HistoryReport.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.Set; + +public class HistoryReport { + private List buildsWithTestResults; + private Map historyResults; + private Map> skippedTests; + + public HistoryReport() { + buildsWithTestResults = new ArrayList(); + this.historyResults = new HashMap(); + } + + public Map getHistoryResults() { + return this.historyResults; + } + + public Map> getSkippedTests() { + return this.skippedTests; + } + + public List getBuildsWithTestResults() { + return this.buildsWithTestResults; + } + + public void setBuildsWithTestResults(List src) { + this.buildsWithTestResults = src; + } + + public void setHistoryResults(Map src, Map> skippedTests) { + this.skippedTests = skippedTests; + this.historyResults = src; + } + + public void printReport() { + System.out.printf("%-30s", "Failed Test Cases Stats"); + for (Integer i : getBuildsWithTestResults()) { + System.out.printf("%5d", i); + } + System.out.println("\n========================================================"); + SortedSet keys = new TreeSet(getHistoryResults().keySet()); + for (String failedTestCase : keys) { + System.out.println(); + int[] resultHistory = getHistoryResults().get(failedTestCase); + System.out.print(failedTestCase); + for (int i = 0; i < resultHistory.length; i++) { + System.out.printf("%5d", resultHistory[i]); + } + } + System.out.println(); + + if (skippedTests == null) return; + + System.out.printf("\n%-30s\n", "Skipped Test Cases Stats"); + for (Integer i : getBuildsWithTestResults()) { + Set tmpSkippedTests = skippedTests.get(i); + if (tmpSkippedTests == null || tmpSkippedTests.isEmpty()) continue; + System.out.printf("======= %d skipped(Or don't have) following test suites =======\n", i); + for (String skippedTestcase : tmpSkippedTests) { + System.out.println(skippedTestcase); + } + } + } +} \ No newline at end of file diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java new file mode 100644 index 0000000..e476cb9 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestCaseResult.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +public class TestCaseResult { + private String className; + private int failedSince; + private String name; + private String status; + + public String getName() { + return name; + } + + public String getClassName() { + return className; + } + + public int failedSince() { + return failedSince; + } + + public String getStatus() { + return status; + } + + public void setName(String s) { + name = s; + } + + public void setClassName(String s) { + className = s; + } + + public void setFailedSince(int s) { + failedSince = s; + } + + public void setStatus(String s) { + status = s; + } + + public String getFullName() { + return (this.className + "." + this.name).toLowerCase(); + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java new file mode 100644 index 0000000..0270f91 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestResultHistory.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.JenkinsServer; +import com.offbytwo.jenkins.client.JenkinsHttpClient; +import com.offbytwo.jenkins.model.*; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + +public class TestResultHistory { + public final static String STATUS_REGRESSION = "REGRESSION"; + public final static String STATUS_FAILED = "FAILED"; + public final static String STATUS_PASSED = "PASSED"; + public final static String STATUS_FIXED = "FIXED"; + public static int BUILD_HISTORY_NUM = 15; + + private JenkinsHttpClient client; + private String jobName; + + public TestResultHistory(String apacheHTTPURL, String jobName, String userName, String passWord) + throws URISyntaxException { + this.client = new JenkinsHttpClient(new URI(apacheHTTPURL), userName, passWord); + this.jobName = jobName; + } + + public static void main(String[] args) { + + if (args.length < 2) { + printUsage(); + return; + } + + String apacheHTTPUrl = args[0]; + String jobName = args[1]; + if (args.length > 2) { + int tmpHistoryJobNum = -1; + try { + tmpHistoryJobNum = Integer.parseInt(args[2]); + } catch (NumberFormatException ex) { + // ignore + } + if (tmpHistoryJobNum > 0) { + BUILD_HISTORY_NUM = tmpHistoryJobNum; + } + } + + try { + TestResultHistory buildHistory = new TestResultHistory(apacheHTTPUrl, jobName, "", ""); + HistoryReport report = buildHistory.getReport(); + // display result in console + report.printReport(); + } catch (Exception ex) { + System.out.println("Got unexpected exception: " + ex.getMessage()); + } + } + + protected static void printUsage() { + System.out.println(" [Number of Historical Jobs to Check]"); + System.out.println("Sample Input: \"https://builds.apache.org\" " + + "\"HBase-TRUNK-on-Hadoop-2.0.0\" "); + } + + public HistoryReport getReport() { + HistoryReport report = new HistoryReport(); + + List buildWithTestResults = new ArrayList(); + Map failureStats = new HashMap(); + + try { + JenkinsServer jenkins = new JenkinsServer(this.client); + Map jobs = jenkins.getJobs(); + JobWithDetails job = jobs.get(jobName.toLowerCase()).details(); + + // build test case failures stats for the past 10 builds + Build lastBuild = job.getLastBuild(); + int startingBuildNumber = + (lastBuild.getNumber() - BUILD_HISTORY_NUM > 0) ? lastBuild.getNumber() + - BUILD_HISTORY_NUM + 1 : 1; + + Map> executedTestCases = + new HashMap>(); + Map> skippedTestCases = new TreeMap>(); + Set allExecutedTestCases = new HashSet(); + Map> normalizedTestSet = new HashMap>(); + String buildUrl = lastBuild.getUrl(); + for (int i = startingBuildNumber; i <= lastBuild.getNumber(); i++) { + HashMap buildExecutedTestCases = new HashMap(2048); + String curBuildUrl = buildUrl.replaceFirst("/" + lastBuild.getNumber(), "/" + i); + List failedCases = null; + try { + failedCases = getBuildFailedTestCases(curBuildUrl, buildExecutedTestCases); + buildWithTestResults.add(i); + } catch (Exception ex) { + // can't get result so skip it + continue; + } + executedTestCases.put(i, buildExecutedTestCases); + HashSet tmpSet = new HashSet(); + for (String tmpTestCase : buildExecutedTestCases.keySet()) { + allExecutedTestCases.add(tmpTestCase.substring(0, tmpTestCase.lastIndexOf("."))); + tmpSet.add(tmpTestCase.substring(0, tmpTestCase.lastIndexOf("."))); + } + normalizedTestSet.put(i, tmpSet); + + // set test result failed cases of current build + for (String curFailedTestCase : failedCases) { + if (failureStats.containsKey(curFailedTestCase)) { + int[] testCaseResultArray = failureStats.get(curFailedTestCase); + testCaseResultArray[i - startingBuildNumber] = -1; + } else { + int[] testResult = new int[BUILD_HISTORY_NUM]; + testResult[i - startingBuildNumber] = -1; + // refill previous build test results for newly failed test case + for (int k = startingBuildNumber; k < i; k++) { + HashMap tmpBuildExecutedTestCases = executedTestCases.get(k); + if (tmpBuildExecutedTestCases != null + && tmpBuildExecutedTestCases.containsKey(curFailedTestCase)) { + String statusStr = tmpBuildExecutedTestCases.get(curFailedTestCase); + testResult[k - startingBuildNumber] = convertStatusStringToInt(statusStr); + } + } + failureStats.put(curFailedTestCase, testResult); + } + + } + + // set test result for previous failed test cases + for (String curTestCase : failureStats.keySet()) { + if (!failedCases.contains(curTestCase) && buildExecutedTestCases.containsKey(curTestCase)) { + String statusVal = buildExecutedTestCases.get(curTestCase); + int[] testCaseResultArray = failureStats.get(curTestCase); + testCaseResultArray[i - startingBuildNumber] = convertStatusStringToInt(statusVal); + } + } + } + + // check which test suits skipped + for (int i = startingBuildNumber; i <= lastBuild.getNumber(); i++) { + Set skippedTests = new HashSet(); + HashMap tmpBuildExecutedTestCases = executedTestCases.get(i); + if (tmpBuildExecutedTestCases == null || tmpBuildExecutedTestCases.isEmpty()) continue; + // normalize test case names + Set tmpNormalizedTestCaseSet = normalizedTestSet.get(i); + for (String testCase : allExecutedTestCases) { + if (!tmpNormalizedTestCaseSet.contains(testCase)) { + skippedTests.add(testCase); + } + } + skippedTestCases.put(i, skippedTests); + } + + report.setBuildsWithTestResults(buildWithTestResults); + for (String failedTestCase : failureStats.keySet()) { + int[] resultHistory = failureStats.get(failedTestCase); + int[] compactHistory = new int[buildWithTestResults.size()]; + int index = 0; + for (Integer i : buildWithTestResults) { + compactHistory[index] = resultHistory[i - startingBuildNumber]; + index++; + } + failureStats.put(failedTestCase, compactHistory); + } + + report.setHistoryResults(failureStats, skippedTestCases); + + } catch (Exception ex) { + System.out.println(ex); + ex.printStackTrace(); + } + + return report; + } + + /** + * @param statusVal + * @return 1 means PASSED, -1 means FAILED, 0 means SKIPPED + */ + static int convertStatusStringToInt(String statusVal) { + + if (statusVal.equalsIgnoreCase(STATUS_REGRESSION) || statusVal.equalsIgnoreCase(STATUS_FAILED)) { + return -1; + } else if (statusVal.equalsIgnoreCase(STATUS_PASSED)) { + return 1; + } + + return 0; + } + + /** + * Get failed test cases of a build + * @param buildURL Jenkins build job URL + * @param executedTestCases Set of test cases which was executed for the build + * @return list of failed test case names + */ + List getBuildFailedTestCases(String buildURL, HashMap executedTestCases) + throws IOException { + List result = new ArrayList(); + + String apiPath = + urlJoin(buildURL, + "testReport?depth=10&tree=suites[cases[className,name,status,failedSince]]"); + + List suites = client.get(apiPath, BuildResultWithTestCaseDetails.class).getSuites(); + + result = getTestSuiteFailedTestcase(suites, executedTestCases); + + return result; + } + + private List getTestSuiteFailedTestcase(List suites, + HashMap executedTestCases) { + List result = new ArrayList(); + + if (suites == null) { + return result; + } + + for (TestSuite curTestSuite : suites) { + for (TestCaseResult curTestCaseResult : curTestSuite.getCases()) { + if (curTestCaseResult.getStatus().equalsIgnoreCase(STATUS_FAILED) + || curTestCaseResult.getStatus().equalsIgnoreCase(STATUS_REGRESSION)) { + // failed test case + result.add(curTestCaseResult.getFullName()); + } + executedTestCases.put(curTestCaseResult.getFullName(), curTestCaseResult.getStatus()); + } + } + + return result; + } + + String urlJoin(String path1, String path2) { + if (!path1.endsWith("/")) { + path1 += "/"; + } + if (path2.startsWith("/")) { + path2 = path2.substring(1); + } + return path1 + path2; + } +} diff --git dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java new file mode 100644 index 0000000..b8a7624 --- /dev/null +++ dev-support/jenkins-tools/buildstats/src/main/java/org/apache/hadoop/hbase/devtools/buildstats/TestSuite.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.devtools.buildstats; + +import com.offbytwo.jenkins.model.BaseModel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class TestSuite extends BaseModel { + List cases; + + public TestSuite() { + this(new ArrayList()); + } + + public TestSuite(List s) { + this.cases = s; + } + + public TestSuite(TestCaseResult... s) { + this(Arrays.asList(s)); + } + + public List getCases() { + return cases; + } + + public void setCases(List s) { + cases = s; + } +} diff --git dev-support/jenkins-tools/pom.xml dev-support/jenkins-tools/pom.xml new file mode 100644 index 0000000..952b29a --- /dev/null +++ dev-support/jenkins-tools/pom.xml @@ -0,0 +1,36 @@ + + + + + 4.0.0 + + org.apache.hbase + jenkins-tools + 1.0 + pom + + + jenkins-client + buildstats + +