Index: ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java =================================================================== --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (revision 957005) +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (working copy) @@ -95,7 +95,17 @@ private String logFile; private String clusterMode; + + private String hadoopVersion; + public void setHadoopVersion(String ver) { + this.hadoopVersion = ver; + } + + public String getHadoopVersion() { + return hadoopVersion; + } + public void setClusterMode(String clusterMode) { this.clusterMode = clusterMode; } @@ -291,6 +301,9 @@ if (clusterMode == null) { clusterMode = new String(""); } + if (hadoopVersion == null) { + hadoopVersion = ""; + } // For each of the qFiles generate the test VelocityContext ctx = new VelocityContext(); @@ -299,6 +312,7 @@ ctx.put("resultsDir", resultsDir); ctx.put("logDir", logDir); ctx.put("clusterMode", clusterMode); + ctx.put("hadoopVersion", hadoopVersion); File outFile = new File(outDir, className + ".java"); FileWriter writer = new FileWriter(outFile); Index: hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java =================================================================== --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (revision 957005) +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (working copy) @@ -27,7 +27,7 @@ String outDir, String logDir, boolean miniMr, HBaseTestSetup setup) throws Exception { - super(outDir, logDir, miniMr); + super(outDir, logDir, miniMr, null); setup.preTest(conf); super.init(); } Index: build-common.xml =================================================================== --- build-common.xml (revision 957005) +++ build-common.xml (working copy) @@ -52,6 +52,7 @@ + Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 957005) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -28,6 +28,7 @@ import java.io.Serializable; import java.net.URI; import java.util.ArrayList; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -96,6 +97,7 @@ private MiniMRCluster mr = null; private HadoopShims.MiniDFSShim dfs = null; private boolean miniMr = false; + private String hadoopVer = null; public boolean deleteDirectory(File path) { if (path.exists()) { @@ -167,14 +169,27 @@ } public QTestUtil(String outDir, String logDir) throws Exception { - this(outDir, logDir, false); + this(outDir, logDir, false, "0.20"); } - public QTestUtil(String outDir, String logDir, boolean miniMr) throws Exception { + private String getHadoopMainVersion(String input) { + if (input == null) { + return null; + } + Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); + Matcher m = p.matcher(input); + if (m.matches()) { + return m.group(1); + } + return null; + } + + public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception { this.outDir = outDir; this.logDir = logDir; conf = new HiveConf(Driver.class); this.miniMr = miniMr; + this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap(); qSkipSet = new HashSet(); @@ -706,6 +721,7 @@ } + public int checkResults(String tname) throws Exception { Path warehousePath = new Path(FileSystem.get(conf).getUri().getPath()); warehousePath = new Path(warehousePath, (new URI(testWarehouse)).getPath()); @@ -783,10 +799,47 @@ return exitVal; } + /** + * Given the current configurations (e.g., hadoop version and execution mode), return + * the correct file name to compare with the current test run output. + * @param outDir The directory where the reference log files are stored. + * @param testName The test file name (terminated by ".out"). + * @return The file name appended with the configuration values if it exists. + */ + public String outPath(String outDir, String testName) { + String ret = testName; + // List of configurations. Currently the list consists of hadoop version and execution mode only + List configs = new ArrayList(); + configs.add(this.hadoopVer); + + Deque stack = new LinkedList(); + StringBuilder sb = new StringBuilder(); + sb.append(testName); + stack.push(sb.toString()); + + // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17 + for (String s: configs) { + sb.append('_'); + sb.append(s); + stack.push(sb.toString()); + } + while (stack.size() > 0) { + String fileName = stack.pop(); + File f = new File(outDir, fileName); + if (f.exists()) { + ret = f.getPath(); + break; + } + } + return ret; + } + public int checkCliDriverResults(String tname) throws Exception { String[] cmdArray; assert(qMap.containsKey(tname)); + String outFileName = outPath(outDir, tname + ".out"); + cmdArray = new String[] { "diff", "-a", "-I", "file:", @@ -804,7 +857,7 @@ "-I", "Caused by:", "-I", "[.][.][.] [0-9]* more", (new File(logDir, tname + ".out")).getPath(), - (new File(outDir, tname + ".out")).getPath()}; + outFileName }; System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); @@ -825,7 +878,7 @@ cmdArray = new String[3]; cmdArray[0] = "cp"; cmdArray[1] = (new File(logDir, tname + ".out")).getPath(); - cmdArray[2] = (new File(outDir, tname + ".out")).getPath(); + cmdArray[2] = outFileName; executor = Runtime.getRuntime().exec(cmdArray); exitVal = executor.waitFor(); } Index: ql/src/test/templates/TestCliDriver.vm =================================================================== --- ql/src/test/templates/TestCliDriver.vm (revision 957005) +++ ql/src/test/templates/TestCliDriver.vm (working copy) @@ -33,10 +33,13 @@ protected void setUp() { try { boolean miniMR = false; + String hadoopVer; + if ("$clusterMode".equals("miniMR")) miniMR = true; + hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR); + qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer); } catch (Exception e) { Index: ql/build.xml =================================================================== --- ql/build.xml (revision 957005) +++ ql/build.xml (working copy) @@ -88,7 +88,9 @@ clusterMode="${clustermode}" resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver" logFile="${test.log.dir}/testclidrivergen.log" - logDirectory="${test.log.dir}/clientpositive"/> + logDirectory="${test.log.dir}/clientpositive" + hadoopVersion="${hadoopVersion}" + />