Index: ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
===================================================================
--- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (revision 956704)
+++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (working copy)
@@ -95,7 +95,17 @@
private String logFile;
private String clusterMode;
+
+ private String hadoopVersion;
+ public void setHadoopVersion(String ver) {
+ this.hadoopVersion = ver;
+ }
+
+ public String getHadoopVersion() {
+ return hadoopVersion;
+ }
+
public void setClusterMode(String clusterMode) {
this.clusterMode = clusterMode;
}
@@ -291,6 +301,9 @@
if (clusterMode == null) {
clusterMode = new String("");
}
+ if (hadoopVersion == null) {
+ hadoopVersion = new String("");
+ }
// For each of the qFiles generate the test
VelocityContext ctx = new VelocityContext();
@@ -299,6 +312,7 @@
ctx.put("resultsDir", resultsDir);
ctx.put("logDir", logDir);
ctx.put("clusterMode", clusterMode);
+ ctx.put("hadoopVersion", hadoopVersion);
File outFile = new File(outDir, className + ".java");
FileWriter writer = new FileWriter(outFile);
Index: build.properties
===================================================================
--- build.properties (revision 956704)
+++ build.properties (working copy)
@@ -18,6 +18,7 @@
hadoop.version.ant-internal=${hadoop.version}
+hadoop.mode.ant-internal=${hadoop.mode}
hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
hadoop.root=${hadoop.root.default}
hadoop.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
Index: hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
===================================================================
--- hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (revision 956704)
+++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (working copy)
@@ -27,7 +27,7 @@
String outDir, String logDir, boolean miniMr, HBaseTestSetup setup)
throws Exception {
- super(outDir, logDir, miniMr);
+ super(outDir, logDir, miniMr, null);
setup.preTest(conf);
super.init();
}
Index: hbase-handler/build.xml
===================================================================
--- hbase-handler/build.xml (revision 956704)
+++ hbase-handler/build.xml (working copy)
@@ -73,7 +73,9 @@
clusterMode="${clustermode}"
resultsDirectory="${hbase-handler.test.results.dir}" className="TestHBaseCliDriver"
logFile="${test.log.dir}/testhbaseclidrivergen.log"
- logDirectory="${test.log.dir}/hbase-handler"/>
+ logDirectory="${test.log.dir}/hbase-handler"
+ hadoopVersion="${hadoop.version.ant-internal.prefix}"
+ />
+ logDirectory="${test.log.dir}/hbase-handler"
+ hadoopVersion="${hadoop.version.ant-internal.prefix}"
+ />
+
+
+
+
+
Index: build-common.xml
===================================================================
--- build-common.xml (revision 956704)
+++ build-common.xml (working copy)
@@ -52,6 +52,8 @@
+
+
Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
===================================================================
--- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 956704)
+++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy)
@@ -33,6 +33,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.Stack;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -96,6 +97,7 @@
private MiniMRCluster mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private boolean miniMr = false;
+ private String hadoopVer = null;
public boolean deleteDirectory(File path) {
if (path.exists()) {
@@ -167,14 +169,27 @@
}
public QTestUtil(String outDir, String logDir) throws Exception {
- this(outDir, logDir, false);
+ this(outDir, logDir, false, "0.20");
}
- public QTestUtil(String outDir, String logDir, boolean miniMr) throws Exception {
+ private String getHadoopMainVersion(String input) {
+ if (input == null) {
+ return null;
+ }
+ Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");
+ Matcher m = p.matcher(input);
+ if (m.matches()) {
+ return m.group(1);
+ }
+ return null;
+ }
+
+ public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception {
this.outDir = outDir;
this.logDir = logDir;
conf = new HiveConf(Driver.class);
this.miniMr = miniMr;
+ this.hadoopVer = getHadoopMainVersion(hadoopVer);
qMap = new TreeMap();
qSkipSet = new HashSet();
@@ -706,6 +721,7 @@
}
+
public int checkResults(String tname) throws Exception {
Path warehousePath = new Path(FileSystem.get(conf).getUri().getPath());
warehousePath = new Path(warehousePath, (new URI(testWarehouse)).getPath());
@@ -783,10 +799,48 @@
return exitVal;
}
+ /**
+ * Given the current configurations (e.g., hadoop version and execution mode), return
+ * the correct file name to compare with the current test run output.
+ * @param outDir The directory where the reference log files are stored.
+ * @param testName The test file name (terminated by ".out").
+ * @return The file name appended with the configuration values if it exists.
+ */
+ public String outPath(String outDir, String testName) {
+ String ret = testName;
+ // List of configurations. Currently the list consists of hadoop version and execution mode only
+ List configs = new ArrayList();
+ configs.add(this.hadoopVer);
+ configs.add(miniMr? "miniMR": "local");
+
+ Stack stack = new Stack();
+ StringBuilder sb = new StringBuilder();
+ sb.append(testName);
+ stack.push(sb.toString());
+
+ // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17
+ for (String s: configs) {
+ sb.append('_');
+ sb.append(s);
+ stack.push(sb.toString());
+ }
+ while (stack.size() > 0) {
+ String fileName = stack.pop();
+ File f = new File(outDir, fileName);
+ if (f.exists()) {
+ ret = f.getPath();
+ break;
+ }
+ }
+ return ret;
+ }
+
public int checkCliDriverResults(String tname) throws Exception {
String[] cmdArray;
assert(qMap.containsKey(tname));
+ String outFileName = outPath(outDir, tname + ".out");
+
cmdArray = new String[] {
"diff", "-a",
"-I", "file:",
@@ -804,7 +858,7 @@
"-I", "Caused by:",
"-I", "[.][.][.] [0-9]* more",
(new File(logDir, tname + ".out")).getPath(),
- (new File(outDir, tname + ".out")).getPath()};
+ outFileName };
System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' '));
@@ -825,7 +879,7 @@
cmdArray = new String[3];
cmdArray[0] = "cp";
cmdArray[1] = (new File(logDir, tname + ".out")).getPath();
- cmdArray[2] = (new File(outDir, tname + ".out")).getPath();
+ cmdArray[2] = outFileName;
executor = Runtime.getRuntime().exec(cmdArray);
exitVal = executor.waitFor();
}
Index: ql/src/test/templates/TestCliDriver.vm
===================================================================
--- ql/src/test/templates/TestCliDriver.vm (revision 956704)
+++ ql/src/test/templates/TestCliDriver.vm (working copy)
@@ -33,10 +33,13 @@
protected void setUp() {
try {
boolean miniMR = false;
+ String hadoopVer;
+
if ("$clusterMode".equals("miniMR"))
miniMR = true;
+ hadoopVer = "$hadoopVersion";
- qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR);
+ qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer);
}
catch (Exception e) {
Index: ql/build.xml
===================================================================
--- ql/build.xml (revision 956704)
+++ ql/build.xml (working copy)
@@ -69,7 +69,9 @@
queryFileRegex="${qfile_regex}"
resultsDirectory="${ql.test.results.dir}/compiler" className="TestParse"
logFile="${test.log.dir}/testparsegen.log"
- logDirectory="${test.log.dir}/positive"/>
+ logDirectory="${test.log.dir}/positive"
+ clusterMode="${clusterMode}"
+ />
+ logDirectory="${test.log.dir}/negative"
+ clusterMode="${clusterMode}"
+ />
+ logDirectory="${test.log.dir}/clientpositive"
+ hadoopVersion="${hadoopVersion}"
+ />
+ logDirectory="${test.log.dir}/clientnegative"
+ clusterMode="${clusterMode}"
+ />