From b7a1db601e27c97346df963348a170edf6c9c518 Mon Sep 17 00:00:00 2001
From: Greg Bowyer <gbowyer@apache.org>
Date: Fri, 24 Aug 2012 20:42:37 -0700
Subject: [PATCH] LUCENE-4332: Integrate PiTest mutation coverage tool into
 build

---
 build.xml                          |    9 ++
 lucene/build.xml                   |    8 ++
 lucene/common-build.xml            |  145 +++++++++++++++++++++++++++++++++++-
 lucene/test-framework/ivy.xml      |    1 +
 lucene/tools/build.xml             |    1 +
 solr/build.xml                     |   19 +++++
 solr/example/build.xml             |    3 +
 solr/example/example-DIH/build.xml |    3 +
 8 files changed, 188 insertions(+), 1 deletions(-)

diff --git a/build.xml b/build.xml
index 880b545..97a9874 100644
--- a/build.xml
+++ b/build.xml
@@ -33,6 +33,15 @@
     </sequential>
   </target>
 
+  <target name="pitest" description="Test both Lucene and Solr">
+    <sequential>
+      <subant target="pitest" inheritall="false" failonerror="false">
+        <fileset dir="lucene" includes="build.xml" />
+        <fileset dir="solr" includes="build.xml" />
+      </subant>
+    </sequential>
+  </target>
+
   <target name="javadocs" description="Generate Lucene and Solr javadocs">
     <sequential>
       <subant target="javadocs" inheritall="false" failonerror="true">
diff --git a/lucene/build.xml b/lucene/build.xml
index d6ed807..52f4239 100644
--- a/lucene/build.xml
+++ b/lucene/build.xml
@@ -55,6 +55,10 @@
           description="Runs all unit tests (core, modules and back-compat)"
   />
 
+  <target name="pitest" depends="pitest-modules"
+          description="Runs pitests (core, modules and back-compat)"
+  />
+
   <path id="backwards.test.compile.classpath">
     <path refid="junit-path"/>
     <path refid="ant-path"/>
@@ -490,6 +494,10 @@
     <modules-crawl target="test" failonerror="true"/>
   </target>
 
+  <target name="pitest-modules" depends="compile-test">
+    <modules-crawl target="pitest" failonerror="false"/>
+  </target>
+
   <!--
    compile changes.txt into an html file
    -->
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index f3634ab..8f53123 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -106,7 +106,7 @@
     <isset property="run.clover"/>
   </condition>
   <property name="tests.clover.args" value=""/>
-  
+
   <property name="tests.tempDir" value="${build.dir}/test"/>
 
   <property name="tests.cachefile" location="${common.dir}/tools/junit4/cached-timehints.txt" />
@@ -225,6 +225,15 @@
   <property name="clover.db.dir" location="${common.dir}/build/clover/db"/>
   <property name="clover.report.dir" location="${common.dir}/build/clover/reports"/>
 
+  <property name="pitest.report.dir" location="${common.dir}/build/pitest/${name}/reports"/>
+  <property name="pitest.distance" value="0" />
+  <property name="pitest.threads" value="2" />
+  <property name="pitest.testCases" value="org.apache.*" />
+  <property name="pitest.maxMutations" value="0" />
+  <property name="pitest.timeoutFactor" value="1.25" />
+  <property name="pitest.timeoutConst" value="3000" />
+  <property name="pitest.targetClasses" value="org.apache.*" />
+
   <!-- a reasonable default exclusion set, can be overridden for special cases -->
   <property name="rat.excludes" value="**/TODO,**/*.txt"/>
 
@@ -1157,6 +1166,16 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
     <path id="clover.classpath"/>
   </target>
 
+  <target name="pitest" if="run.pitest" depends="compile-test,install-junit4-taskdef,clover,validate"
+      description="Run Unit tests using pitest mutation testing. To use, specify -Drun.pitest=true on the command line.">
+    <echo>Code coverage with pitest enabled.</echo>
+    <ivy:cachepath
+        organisation="org.pitest" module="pitest-ant"
+        inline="true"
+        pathid="pitest.framework.classpath" />
+    <pitest-macro />
+  </target>
+
   <target name="generate-test-reports" description="Generates test reports">
     <mkdir dir="${junit.reports}"/>
     <junitreport todir="${junit.output.dir}">
@@ -1762,4 +1781,128 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
      </sequential>
   </macrodef>
 
+  <macrodef name="pitest-macro" description="Executes junit tests.">
+    <attribute name="pitest.report.dir" default="${pitest.report.dir}"/>
+    <attribute name="pitest.framework.classpath" default="pitest.framework.classpath"/>
+    <attribute name="pitest.distance" default="${pitest.distance}" />
+    <attribute name="pitest.sysprops" default="${pitest.sysprops}" />
+    <attribute name="pitest.threads" default="${pitest.threads}" />
+    <attribute name="pitest.testCases" default="${pitest.testCases}" />
+    <attribute name="pitest.maxMutations" default="${pitest.maxMutations}" />
+    <attribute name="pitest.timeoutFactor" default="${pitest.timeoutFactor}" />
+    <attribute name="pitest.timeoutConst" default="${pitest.timeoutConst}" />
+    <attribute name="pitest.targetClasses" default="${pitest.targetClasses}" />
+
+    <attribute name="junit.classpath" default="junit.classpath"/>
+
+    <attribute name="src.dir" default="${src.dir}"/>
+    <attribute name="build.dir" default="${build.dir}"/>
+
+    <sequential>
+
+        <echo>
+PiTest mutation coverage can take a *long* time on even large hardware.
+(EC2 32core sandy bridge takes at least 6 hours to run PiTest for the lucene test cases)
+
+The following arguments can be provided to ant to alter its behaviour and target specific tests::
+
+-Dpitest.report.dir (@{pitest.report.dir}) - Change where PiTest writes output reports
+
+-Dpitest.distance (@{pitest.distance}) - How far away from the test class should be mutated
+   0 being immeditate callees only
+
+-Dpitest.threads (@{pitest.threads}) - How many threads to use in PiTest 
+   (note this is independent of junit threads)
+
+-Dpitest.testCases (@{pitest.testCases}) - Glob of testcases to run
+
+-Dpitest.maxMutations (@{pitest.maxMutations}) - Maximum number of mutations per class under test
+    0 being unlimited
+
+-Dpitest.timeoutFactor (@{pitest.timeoutFactor}) - Tunable factor used to determine
+    if a test is potentially been mutated to be an infinate loop or O(n!) (or similar)
+
+-Dpitest.timeoutConst (@{pitest.timeoutConst}) - Base constant used for working out timeouts
+
+-Dpitest.targetClasses (@{pitest.targetClasses}) - Classes to consider for mutation
+        </echo>
+
+        <taskdef name="pitest" classname="org.pitest.ant.PitestTask"
+            classpathref="pitest.framework.classpath" />
+
+        <path id="pitest.classpath">
+            <path refid="junit.classpath"/>
+            <path refid="pitest.framework.classpath"/>
+            <pathelement path="${java.class.path}"/>
+        </path>
+
+        <!-- Java7 has a new bytecode verifier that _requires_ stackmaps to be present in the
+             bytecode. Unfortunately a *lot* of bytecode manipulation tools (including pitest)
+             do not currently write out this information; for now we are forced to disable this
+             in the jvm args -->
+        <condition property="pitest.extra.jvm.args" value="-XX:-UseSplitVerifier">
+            <and>
+                <!-- Hotspot goes by a great many names. Other VMS (jrockit, J9 etc might require other tricks here) -->
+                <or>
+                    <!-- Oracle / sun JVM release -->
+                    <contains string="${java.vm.name}" substring="HotSpot" casesensitive="false" />
+
+                    <!-- Opensource HotSpot (sorta hotspot) -->
+                    <contains string="${java.vm.name}" substring="OpenJDK" casesensitive="false" />
+
+                    <!-- Redhats IcedTea (mostly hotspot) -->
+                    <contains string="${java.vm.name}" substring="IcedTea" casesensitive="false" />
+
+                    <!-- Azul MRI-J (kinda mostly hotspot) -->
+                    <contains string="${java.vm.name}" substring="MRI-J" casesensitive="false" />
+
+                    <!-- Another Azul name (sorta kinda mostly hotspot) -->
+                    <contains string="${java.vm.name}" substring="AVM" casesensitive="false" />
+
+                    <!-- I am sure I have missed some version of hotspot somewhere here -->
+                </or>
+
+                <matches string="${ant.java.version}" pattern="1.7|1.8" />
+            </and>
+        </condition>
+
+        <junit4:pickseed property="pitest.seed" />
+
+        <!-- Sigh string concat is nasty in ant -->
+        <script language="javascript"><![CDATA[
+            let format = Packages.java.lang.String.format;
+
+            let commonDir = project.getProperty("common.dir");
+            let props = {
+                'lucene.version': project.getProperty("dev.version"),
+                'test.seed': project.getProperty("pitest.seed"),
+                'java.security.manager': 'java.lang.SecurityManager',
+                'java.security.policy': commonDir + '/tools/junit4/tests.policy',
+                'java.io.tmpdir': project.getProperty('tests.tempDir'),
+                'tests.sandbox.dir': project.getProperty("build.dir")
+            };
+
+            let sysprops = [format("-D%s=%s", k, v) for ([k, v] in Iterator(props))].join(',');
+            project.setProperty("pitest.sysprops", sysprops);
+        ]]></script>
+
+        <echo>${pitest.sysprops}</echo>
+
+        <pitest
+            classPath="pitest.classpath"
+            targetClasses="@{pitest.targetClasses}"
+            targetTests="@{pitest.testCases}"
+            reportDir="@{pitest.report.dir}"
+            sourceDir="@{src.dir}"
+            threads="@{pitest.threads}"
+            maxMutationsPerClass="@{pitest.maxMutations}"
+            timeoutFactor="@{pitest.timeoutFactor}"
+            timeoutConst="@{pitest.timeoutConst}"
+            verbose="false"
+            dependencyDistance="@{pitest.distance}"
+            mutableCodePaths="@{build.dir}/classes/java"
+            jvmArgs="-ea,${pitest.extra.jvm.args},@{pitest.sysprops}" />
+    </sequential>
+  </macrodef>
+
 </project>
diff --git a/lucene/test-framework/ivy.xml b/lucene/test-framework/ivy.xml
index 6d55434..72c92cc 100644
--- a/lucene/test-framework/ivy.xml
+++ b/lucene/test-framework/ivy.xml
@@ -33,6 +33,7 @@
       <dependency org="org.apache.ant" name="ant-junit" rev="1.8.2" transitive="false" />
 
       <dependency org="junit" name="junit" rev="4.10" transitive="false" conf="default->*;junit4-stdalone->*" />
+
       <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.0.rc5" transitive="false" conf="default->*;junit4-stdalone->*" />
       <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.0.rc5" transitive="false" conf="default->*;junit4-stdalone->*" />
 
diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml
index 106c2da..60f9c51 100644
--- a/lucene/tools/build.xml
+++ b/lucene/tools/build.xml
@@ -45,4 +45,5 @@
   </target>
 
   <target name="javadocs"/> <!-- to make common-build.xml happy -->
+  <target name="pitest"/> <!-- to make common-build.xml happy -->
 </project>
diff --git a/solr/build.xml b/solr/build.xml
index 5764f7b..c687d8c 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -133,6 +133,8 @@
           depends="test-core, test-contrib"/>
   <target name="test-core" description="Runs the core and solrj unit tests."
           depends="test-solr-core, test-solrj"/>
+  <target name="pitest" description="Validate, then run core, solrj, and contrib unit tests."
+          depends="pitest-core, pitest-contrib"/>
   <target name="compile-test" description="Compile unit tests."
           depends="compile-solr-test-framework, compile-test-solr-core, compile-test-solrj, compile-test-contrib"/>
   <target name="javadocs" description="Calls javadocs-all, javadocs-solrj, and javadocs-test-framework"
@@ -162,6 +164,23 @@
   <target name="test-contrib" description="Run contrib unit tests.">
     <contrib-crawl target="test" failonerror="true"/>
   </target>
+
+  <!-- Pitest targets -->
+  <target name="pitest-core" description="PiTest solr core">
+    <ant dir="core" target="pitest" inheritAll="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+  </target>
+
+  <target name="pitest-solrj" description="PiTest java client">
+    <ant dir="solrj" target="pitest" inheritAll="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+  </target>
+
+  <target name="pitest-contrib" description="Run contrib PiTests.">
+    <contrib-crawl target="pitest" failonerror="false"/>
+  </target>
   
   <!-- test-framework targets -->
   <target name="javadocs-test-framework">
diff --git a/solr/example/build.xml b/solr/example/build.xml
index fdfbe95..91ef7b4 100644
--- a/solr/example/build.xml
+++ b/solr/example/build.xml
@@ -33,6 +33,9 @@
   <target name="compile-core"/>
   <target name="compile-test"/>
 
+  <!-- nothing to cover -->
+  <target name="pitest"/>
+
   <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
     <sequential>
     <!-- jetty libs in lib/ -->
diff --git a/solr/example/example-DIH/build.xml b/solr/example/example-DIH/build.xml
index b48b149..fb82edf 100644
--- a/solr/example/example-DIH/build.xml
+++ b/solr/example/example-DIH/build.xml
@@ -35,4 +35,7 @@
   <target name="compile-core"/>
   <target name="compile-test"/>
 
+  <!-- nothing to cover -->
+  <target name="pitest"/>
+
 </project>
-- 
1.7.8.6