Index: summary-reporter/src/org/apache/harmony/eut/reporter/EUTReporter.java =================================================================== --- summary-reporter/src/org/apache/harmony/eut/reporter/EUTReporter.java (revision 577804) +++ summary-reporter/src/org/apache/harmony/eut/reporter/EUTReporter.java (working copy) @@ -172,7 +172,7 @@ // caclulate summary statictics EUTSummaryInfo esi = new EUTSummaryInfo(); esi.ss = new EUTSuiteInfo(); // Summary Suite - esi.crashed_suites = new ArrayList(); + esi.suites_unexpected_crashed = new ArrayList(); for (int i = 0; i < suiteList.size(); i++) { EUTSuiteInfo si = suiteList.get(i); @@ -191,13 +191,24 @@ si.tests_unexpected_end_with_failure; if (si.wasRun) { + + // if suite was crashed then we remove 0 value esi.tests_run_total += si.tests_total; esi.tests_run_total -= si.tests_expected_failures_errors; if (si.isCrashed) { esi.tests_crashed_total += si.tests_total; - esi.suites_crashed_total++; - esi.crashed_suites.add(si); + + // the crash may be expected + if (eflList.indexOf(si.name) != -1) { + si.isCrashExpected = true; + + // must not count expected crashed suite tests number + esi.tests_run_total -= si.tests_total; + } else { + esi.tests_unexpected_crashed_total += si.tests_total; + esi.suites_unexpected_crashed.add(si); + } } } } @@ -225,6 +236,7 @@ System.out.println("EUT SCRIPT: " + "EUT summary report was successfully generated"); + // produce an exit code based on EUT run results if (esi.tests_run_total == esi.ss.tests_reported_passed) { System.out.println("EUT SCRIPT: " + "No unexpected EUT issues detected"); Index: summary-reporter/src/org/apache/harmony/eut/reporter/EUTSuiteInfo.java =================================================================== --- summary-reporter/src/org/apache/harmony/eut/reporter/EUTSuiteInfo.java (revision 577804) +++ summary-reporter/src/org/apache/harmony/eut/reporter/EUTSuiteInfo.java (working copy) @@ -50,6 +50,11 @@ */ boolean isCrashed; + /** + * The crashed suite may be expected according to EFL. + */ + boolean isCrashExpected; + /** * Shows if the suite is available for platform the program is run on. * Like org.eclipse.jdt.core.tests.eval.TestAll is available for Windows @@ -165,10 +170,10 @@ */ final class EUTSummaryInfo { EUTSuiteInfo ss; // Summary Suite - int tests_run_total = 0; // need for Relative Summary - int tests_crashed_total = 0; - int suites_crashed_total = 0; - ArrayList crashed_suites; + int tests_run_total; // need for Relative Summary + int tests_crashed_total; + int tests_unexpected_crashed_total; + ArrayList suites_unexpected_crashed; String eut_version; String os, ws, arch; String tested_vm_ags; Index: summary-reporter/src/org/apache/harmony/eut/reporter/EUTTXTReportEmitter.java =================================================================== --- summary-reporter/src/org/apache/harmony/eut/reporter/EUTTXTReportEmitter.java (revision 577804) +++ summary-reporter/src/org/apache/harmony/eut/reporter/EUTTXTReportEmitter.java (working copy) @@ -50,7 +50,7 @@ out.println("Total run tests : " + esi.tests_run_total); out.println("Relative passrate : " + EUTReporter.makePassrateString( esi.ss.tests_reported_passed, esi.tests_run_total)); - out.println("Unexpected crashes : " + esi.suites_crashed_total); + out.println("Unexpected crashes : " + esi.suites_unexpected_crashed.size()); out.println("Unexpected errors : " + esi.ss.tests_unexpected_end_with_error); out.println("Unexpected failures : " @@ -76,15 +76,16 @@ } private static void emitCrashesResults(EUTSummaryInfo esi) { - if (esi.crashed_suites.size() == 0) { + if (esi.suites_unexpected_crashed.size() == 0) { return; } out.println(); out.println("================================================"); - out.println("Unexpected crashes : " + esi.suites_crashed_total); + out.println("Unexpected crashes : " + + esi.suites_unexpected_crashed.size()); - for (int i = 0; i < esi.crashed_suites.size(); i++) { - EUTSuiteInfo si = esi.crashed_suites.get(i); + for (int i = 0; i < esi.suites_unexpected_crashed.size(); i++) { + EUTSuiteInfo si = esi.suites_unexpected_crashed.get(i); out.println(si.name + "(" + si.tests_total + ")"); } } @@ -117,7 +118,7 @@ continue; } out.println(); - out.println(ti.testClass + ti.testName); + out.println(ti.testClass + '.' + ti.testName); out.println(ti.testIssueMessage); out.println(ti.testIssueContent.toString().trim()); } Index: summary-reporter/src/org/apache/harmony/eut/reporter/EUTHTMLSummaryEmitter.java =================================================================== --- summary-reporter/src/org/apache/harmony/eut/reporter/EUTHTMLSummaryEmitter.java (revision 577804) +++ summary-reporter/src/org/apache/harmony/eut/reporter/EUTHTMLSummaryEmitter.java (working copy) @@ -31,9 +31,8 @@ emitiHTMLheaderAndBodyStart(esi.eut_version); emitRelativeSummary(esi.ss, esi.tests_run_total, - esi.tests_crashed_total); - emitAbsoluteSummary(esi.ss, esi.tests_run_total, - esi.tests_crashed_total); + esi.tests_unexpected_crashed_total); + emitAbsoluteSummary(esi.ss, esi.tests_crashed_total); emitNoteOfErrorFailures(); emitNoteOfSummary(); emitSuitesStatictics(); @@ -116,16 +115,16 @@ } private static void emitRelativeSummary(EUTSuiteInfo ss, - int tests_run_total, int tests_crashed_total) { + int tests_run_total, int tests_unexpected_crashed_total) { emitSummary("Relative Summary", "Sudden", tests_run_total, ss.tests_reported_passed, ss.tests_unexpected_end_with_failure, ss.tests_unexpected_end_with_error, - tests_crashed_total); + tests_unexpected_crashed_total); } private static void emitAbsoluteSummary(EUTSuiteInfo ss, - int tests_run_total, int tests_crashed_total) { + int tests_crashed_total) { emitSummary("Absolute Summary", "Total", ss.tests_total, ss.tests_reported_passed, ss.tests_reported_end_with_failure, @@ -189,7 +188,7 @@ String strCrash = " "; // detect report style based on issue type - if (si.isCrashed) { + if (si.isCrashed && !si.isCrashExpected) { tClass = "Error"; strError = " "; strFailure = " "; @@ -200,6 +199,10 @@ tClass = "Failure"; } + if (si.isCrashed && si.isCrashExpected) { + strCrash = "expected crash"; + } + // emit html code finally out.println(" "); Index: eut3.2.efl =================================================================== --- eut3.2.efl (revision 577804) +++ eut3.2.efl (working copy) @@ -6,14 +6,14 @@ # index.htm files by ./build.xml using information from # results//results/xml/*.xml and results//output.txt. # -# The following rule is used to create this file: +# The following rules are used to create this file: # +# - specify test suite name to expect failure for all tests in suite, like +# "org.eclipse.jdt.ui.tests.LeakTestSuite" +# # - specify particular test name to expect its failure, the test name # is to be taken from results\\results\xml\*.xml, like # "org.eclipse.jdt.core.tests.compiler.regression.LookupTest.test044 - 1.4" -# -# - no expected crashes is possible - there shoould be no crashes -# ############################################################################### # Index: eut3.3.efl =================================================================== --- eut3.3.efl (revision 577804) +++ eut3.3.efl (working copy) @@ -6,14 +6,14 @@ # index.htm files by ./build.xml using information from # results//results/xml/*.xml and results//output.txt. # -# The following rule is used to create this file: +# The following rules are used to create this file: # +# - specify test suite name to expect failure for all tests in suite, like +# "org.eclipse.jdt.ui.tests.LeakTestSuite" +# # - specify particular test name to expect its failure, the test name # is to be taken from results\\results\xml\*.xml, like # "org.eclipse.jdt.core.tests.compiler.regression.LookupTest.test044 - 1.4" -# -# - no expected crashes is possible - there shoould be no crashes -# ############################################################################### #