diff --git a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm index dd1ee64..9872291 100644 --- a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm +++ b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm @@ -26,6 +26,7 @@ import java.util.*; import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.session.SessionState; @@ -35,6 +36,12 @@ public class $className extends TestCase { private AccumuloQTestUtil qt; private AccumuloTestSetup setup; + public static class TestAccumuloCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (AccumuloTestSetup)setup)); + } + } + public $className(String name, AccumuloTestSetup setup) { super(name); qt = null; @@ -83,16 +90,12 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); AccumuloTestSetup setup = new AccumuloTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestAccumuloCliDriverAddTestFromQFiles()); return setup; } diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java index ca959c0..60c7764 100644 --- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java +++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ant; +import java.io.BufferedWriter; import java.io.File; import java.io.FileFilter; import java.io.FileWriter; @@ -453,6 +454,29 @@ public void execute() throws BuildException { hadoopVersion = ""; } + File qFileNames = new File(outputDirectory, className + "QFileNames.txt"); + String qFileNamesFile = qFileNames.getCanonicalPath(); + + if (qFileNames.exists()) { + if (!qFileNames.delete()) { + throw new Exception("Could not delete old query file names containing file " + + qFileNamesFile); + } + } + if (!qFileNames.createNewFile()) { + throw new Exception("Could not create query file names containing file " + + qFileNamesFile); + } + + FileWriter fw = new FileWriter(qFileNames.getCanonicalFile()); + BufferedWriter bw = new BufferedWriter(fw); + + for (File qFile: qFiles) { + bw.write(qFile.getName()); + bw.newLine(); + } + bw.close(); + // For each of the qFiles generate the test System.out.println("hiveRootDir = " + hiveRootDir); VelocityContext ctx = new VelocityContext(); @@ -464,6 +488,7 @@ public void execute() throws BuildException { System.out.println("queryDir = " + strQueryDir); ctx.put("queryDir", strQueryDir); ctx.put("qfiles", qFiles); + ctx.put("qFileNamesFile", qFileNamesFile); ctx.put("qfilesMap", qFilesMap); if (resultsDir != null) { ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir)); diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm index 69fad97..de0be32 100644 --- a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm +++ b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm @@ -24,6 +24,7 @@ import junit.framework.TestSuite; import java.io.*; import java.util.*; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; @@ -34,6 +35,12 @@ public class $className extends TestCase { private HBaseQTestUtil qt; private HBaseTestSetup setup; + public static class TestHBaseCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup)); + } + } + public $className(String name, HBaseTestSetup setup) { super(name); qt = null; @@ -84,14 +91,9 @@ public class $className extends TestCase { } TestSuite suite = new TestSuite(); HBaseTestSetup setup = new HBaseTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestHBaseCliDriverAddTestFromQFiles()); return setup; } diff --git a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm index 82f0584..b402585 100644 --- a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm +++ b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm @@ -25,6 +25,7 @@ import junit.framework.TestSuite; import java.io.*; import java.util.*; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; @@ -35,6 +36,12 @@ public class $className extends TestCase { private HBaseQTestUtil qt; private HBaseTestSetup setup; + public static class TestHBaseNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup)); + } + } + public $className(String name, HBaseTestSetup setup) { super(name); qt = null; @@ -83,16 +90,12 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); HBaseTestSetup setup = new HBaseTestSetup(suite); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname", setup)); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, setup, new TestHBaseNegativeCliDriverAddTestFromQFiles()); return setup; } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f6081f4..d1104b3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -55,6 +55,7 @@ import java.util.regex.Pattern; import junit.framework.Assert; +import junit.framework.TestSuite; import org.apache.commons.lang.StringUtils; import org.apache.commons.io.FileUtils; @@ -158,6 +159,10 @@ private final String initScript; private final String cleanupScript; + public interface SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName); + } + static { for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { srcTable = srcTable.trim(); @@ -1992,4 +1997,42 @@ public void failed(Throwable e, String fname, String debugHint) { (command != null ? " running " + command : "") + (debugHint != null ? debugHint : "")); } + + public static void addTestsToSuiteFromQfileNames( + String qFileNamesFile, + Set qFilesToExecute, + TestSuite suite, + Object setup, + SuiteAddTestFunctor suiteAddTestCallback) { + try { + File qFileNames = new File(qFileNamesFile); + FileReader fr = new FileReader(qFileNames.getCanonicalFile()); + BufferedReader br = new BufferedReader(fr); + String fName = null; + + while ((fName = br.readLine()) != null) { + if (fName.isEmpty() || fName.trim().equals("")) { + continue; + } + + int eIdx = fName.indexOf('.'); + + if (eIdx == -1) { + continue; + } + + String tName = fName.substring(0, eIdx); + + if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) { + suiteAddTestCallback.addTestToSuite(suite, setup, tName); + } + } + br.close(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); + } + } } diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm index 8827f12..ae449c5 100644 --- a/ql/src/test/templates/TestCliDriver.vm +++ b/ql/src/test/templates/TestCliDriver.vm @@ -33,6 +33,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -108,15 +114,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestCliDriverAddTestFromQFiles()); suite.addTest(new $className("testCliDriver_shutdown")); return suite; } diff --git a/ql/src/test/templates/TestCompareCliDriver.vm b/ql/src/test/templates/TestCompareCliDriver.vm index 6b367d6..4712708 100644 --- a/ql/src/test/templates/TestCompareCliDriver.vm +++ b/ql/src/test/templates/TestCompareCliDriver.vm @@ -34,6 +34,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestCompareCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testCompareCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -109,18 +115,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); - - - -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if (qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testCompareCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestCompareCliDriverAddTestFromQFiles()); suite.addTest(new $className("testCompareCliDriver_shutdown")); return suite; } diff --git a/ql/src/test/templates/TestNegativeCliDriver.vm b/ql/src/test/templates/TestNegativeCliDriver.vm index 742044a..5f8ee8e 100644 --- a/ql/src/test/templates/TestNegativeCliDriver.vm +++ b/ql/src/test/templates/TestNegativeCliDriver.vm @@ -33,6 +33,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + public static class TestNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testNegativeCliDriver_"+tName)); + } + } + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String initScript = "$initScript"; @@ -95,15 +101,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testNegativeCliDriver_$tname")); - } -#end + + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestNegativeCliDriverAddTestFromQFiles()); suite.addTest(new $className("testNegativeCliDriver_shutdown")); return suite; } diff --git a/ql/src/test/templates/TestParseNegative.vm b/ql/src/test/templates/TestParseNegative.vm index 33b238e..c5e7bdf 100755 --- a/ql/src/test/templates/TestParseNegative.vm +++ b/ql/src/test/templates/TestParseNegative.vm @@ -32,6 +32,12 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; + + public static class TestParseNegativeAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor { + public void addTestToSuite(TestSuite suite, Object setup, String tName) { + suite.addTest(new $className("testParseNegative_"+tName)); + } + } static { @@ -88,16 +94,11 @@ public class $className extends TestCase { } } } + TestSuite suite = new TestSuite(); -#foreach ($qf in $qfiles) - #set ($fname = $qf.getName()) - #set ($eidx = $fname.indexOf('.')) - #set ($tname = $fname.substring(0, $eidx)) - if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) { - suite.addTest(new $className("testParseNegative_$tname")); - } -#end + QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute, + suite, null, new TestParseNegativeAddTestFromQFiles()); suite.addTest(new $className("testParseNegative_shutdown")); return suite; }