Index: ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java =================================================================== --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (revision 956704) +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (working copy) @@ -95,7 +95,17 @@ private String logFile; private String clusterMode; + + private String hadoopVersion; + public void setHadoopVersion(String ver) { + this.hadoopVersion = ver; + } + + public String getHadoopVersion() { + return hadoopVersion; + } + public void setClusterMode(String clusterMode) { this.clusterMode = clusterMode; } @@ -291,6 +301,9 @@ if (clusterMode == null) { clusterMode = new String(""); } + if (hadoopVersion == null) { + hadoopVersion = new String(""); + } // For each of the qFiles generate the test VelocityContext ctx = new VelocityContext(); @@ -299,6 +312,7 @@ ctx.put("resultsDir", resultsDir); ctx.put("logDir", logDir); ctx.put("clusterMode", clusterMode); + ctx.put("hadoopVersion", hadoopVersion); File outFile = new File(outDir, className + ".java"); FileWriter writer = new FileWriter(outFile); Index: build.properties =================================================================== --- build.properties (revision 956704) +++ build.properties (working copy) @@ -18,6 +18,7 @@ hadoop.version.ant-internal=${hadoop.version} +hadoop.mode.ant-internal=${hadoop.mode} hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal} hadoop.root=${hadoop.root.default} hadoop.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar Index: hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java =================================================================== --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (revision 956704) +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (working copy) @@ -27,7 +27,7 @@ String outDir, String logDir, boolean miniMr, HBaseTestSetup setup) throws Exception { - super(outDir, logDir, miniMr); + super(outDir, logDir, miniMr, null); setup.preTest(conf); super.init(); } Index: hbase-handler/build.xml =================================================================== --- hbase-handler/build.xml (revision 956704) +++ hbase-handler/build.xml (working copy) @@ -73,7 +73,9 @@ clusterMode="${clustermode}" resultsDirectory="${hbase-handler.test.results.dir}" className="TestHBaseCliDriver" logFile="${test.log.dir}/testhbaseclidrivergen.log" - logDirectory="${test.log.dir}/hbase-handler"/> + logDirectory="${test.log.dir}/hbase-handler" + hadoopVersion="${hadoop.version.ant-internal.prefix}" + /> + logDirectory="${test.log.dir}/hbase-handler" + hadoopVersion="${hadoop.version.ant-internal.prefix}" + /> + + + + + Index: build-common.xml =================================================================== --- build-common.xml (revision 956704) +++ build-common.xml (working copy) @@ -52,6 +52,8 @@ + + Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 956704) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -33,6 +33,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.Stack; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -96,6 +97,7 @@ private MiniMRCluster mr = null; private HadoopShims.MiniDFSShim dfs = null; private boolean miniMr = false; + private String hadoopVer = null; public boolean deleteDirectory(File path) { if (path.exists()) { @@ -167,14 +169,27 @@ } public QTestUtil(String outDir, String logDir) throws Exception { - this(outDir, logDir, false); + this(outDir, logDir, false, "0.20"); } - public QTestUtil(String outDir, String logDir, boolean miniMr) throws Exception { + private String getHadoopMainVersion(String input) { + if (input == null) { + return null; + } + Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); + Matcher m = p.matcher(input); + if (m.matches()) { + return m.group(1); + } + return null; + } + + public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception { this.outDir = outDir; this.logDir = logDir; conf = new HiveConf(Driver.class); this.miniMr = miniMr; + this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap(); qSkipSet = new HashSet(); @@ -706,6 +721,7 @@ } + public int checkResults(String tname) throws Exception { Path warehousePath = new Path(FileSystem.get(conf).getUri().getPath()); warehousePath = new Path(warehousePath, (new URI(testWarehouse)).getPath()); @@ -783,10 +799,48 @@ return exitVal; } + /** + * Given the current configurations (e.g., hadoop version and execution mode), return + * the correct file name to compare with the current test run output. + * @param outDir The directory where the reference log files are stored. + * @param testName The test file name (terminated by ".out"). + * @return The file name appended with the configuration values if it exists. + */ + public String outPath(String outDir, String testName) { + String ret = testName; + // List of configurations. Currently the list consists of hadoop version and execution mode only + List configs = new ArrayList(); + configs.add(this.hadoopVer); + configs.add(miniMr? "miniMR": "local"); + + Stack stack = new Stack(); + StringBuilder sb = new StringBuilder(); + sb.append(testName); + stack.push(sb.toString()); + + // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17 + for (String s: configs) { + sb.append('_'); + sb.append(s); + stack.push(sb.toString()); + } + while (stack.size() > 0) { + String fileName = stack.pop(); + File f = new File(outDir, fileName); + if (f.exists()) { + ret = f.getPath(); + break; + } + } + return ret; + } + public int checkCliDriverResults(String tname) throws Exception { String[] cmdArray; assert(qMap.containsKey(tname)); + String outFileName = outPath(outDir, tname + ".out"); + cmdArray = new String[] { "diff", "-a", "-I", "file:", @@ -804,7 +858,7 @@ "-I", "Caused by:", "-I", "[.][.][.] [0-9]* more", (new File(logDir, tname + ".out")).getPath(), - (new File(outDir, tname + ".out")).getPath()}; + outFileName }; System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); @@ -825,7 +879,7 @@ cmdArray = new String[3]; cmdArray[0] = "cp"; cmdArray[1] = (new File(logDir, tname + ".out")).getPath(); - cmdArray[2] = (new File(outDir, tname + ".out")).getPath(); + cmdArray[2] = outFileName; executor = Runtime.getRuntime().exec(cmdArray); exitVal = executor.waitFor(); } Index: ql/src/test/templates/TestCliDriver.vm =================================================================== --- ql/src/test/templates/TestCliDriver.vm (revision 956704) +++ ql/src/test/templates/TestCliDriver.vm (working copy) @@ -33,10 +33,13 @@ protected void setUp() { try { boolean miniMR = false; + String hadoopVer; + if ("$clusterMode".equals("miniMR")) miniMR = true; + hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR); + qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer); } catch (Exception e) { Index: ql/build.xml =================================================================== --- ql/build.xml (revision 956704) +++ ql/build.xml (working copy) @@ -69,7 +69,9 @@ queryFileRegex="${qfile_regex}" resultsDirectory="${ql.test.results.dir}/compiler" className="TestParse" logFile="${test.log.dir}/testparsegen.log" - logDirectory="${test.log.dir}/positive"/> + logDirectory="${test.log.dir}/positive" + clusterMode="${clusterMode}" + /> + logDirectory="${test.log.dir}/negative" + clusterMode="${clusterMode}" + /> + logDirectory="${test.log.dir}/clientpositive" + hadoopVersion="${hadoopVersion}" + /> + logDirectory="${test.log.dir}/clientnegative" + clusterMode="${clusterMode}" + />