diff --git ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java index d7abdfd..a90a98b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java @@ -29,6 +29,8 @@ import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; import java.io.Serializable; @@ -78,6 +80,7 @@ import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; @@ -1100,7 +1103,11 @@ private static int executeDiffCommand(String inFileName, diffCommandArgs.add(getQuotedString(inFileName)); diffCommandArgs.add(getQuotedString(outFileName)); - result = executeCmd(diffCommandArgs); + if (Shell.WINDOWS) { + String[] cmdArray = (String[]) diffCommandArgs.toArray(new String[diffCommandArgs.size()]); + result = executeWindowsFilterDiff(cmdArray); + } else + result = executeCmd(diffCommandArgs); if (sortResults) { new File(inFileName).delete(); @@ -1129,7 +1136,106 @@ private static int executeCmd(Collection args, String outFile, String er String[] cmdArray = (String[]) args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } + + //Special StreamPrinter used for filter diff result from TestCliDriver tests on Windows + //Use a flag to indicate if any lines in diff result contain information about real + //error in query execution or just expected result inconsistency between Windows + //and Linux. + public static class CliTestStreamPrinter extends Thread { + InputStream is; + String type; + PrintStream os; + Boolean flag; + String[] patterns; + String maskPattern; + + private Boolean hasUnexpectedDiffs(){ + return this.flag; + } + public CliTestStreamPrinter(InputStream is, String type, PrintStream os, String[] patterns) { + this.is = is; + this.type = type; + this.os = os; + this.flag = false; + this.patterns = patterns; + this.maskPattern = "############maskhere#############"; + } + + @Override + public void run() { + BufferedReader br = null; + try { + InputStreamReader isr = new InputStreamReader(is); + br = new BufferedReader(isr); + String line = null; + if (type != null) { + while ((line = br.readLine()) != null) { + os.println(type + ">" + line); + } + } else { + while ((line = br.readLine()) != null) { + os.println(line); + for (String pattern : patterns) { + line = line.replaceAll(pattern, maskPattern); + } + if (!line.equals(maskPattern)) + flag = true; + } + } + br.close(); + br=null; + } catch (IOException ioe) { + ioe.printStackTrace(); + }finally{ + IOUtils.closeStream(br); + } + } + } + + //executeWindowsFilterDiff is used when TestCliDriver tests run diff on result on Windows + //this function will filter diff output according to the patterns which is + //expected result inconsistency between Linux and Windows (for example, file size + //may different because of different new line character) + private static int executeWindowsFilterDiff(String[] args) throws Exception { + System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); + String[] patterns = new String[] { + "^---$", + "^([0-9]+|([0-9]+,[0-9]+))c([0-9]+|([0-9]+,[0-9]+))$", + "^> *totalSize [0-9]+$", + "^< *totalSize [0-9]+$", + "^> *totalFileSize:[0-9]+$", + "^< *totalFileSize:[0-9]+$", + "^> *maxFileSize:[0-9]+$", + "^< *maxFileSize:[0-9]+$", + "^> *minFileSize:[0-9]+$", + "^< *minFileSize:[0-9]+$" + }; + PrintStream out = SessionState.getConsole().getChildOutStream(); + PrintStream err = SessionState.getConsole().getChildErrStream(); + + Process executor = Runtime.getRuntime().exec(args); + + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); + CliTestStreamPrinter outPrinter = new CliTestStreamPrinter(executor.getInputStream(), null, out, patterns); + + outPrinter.start(); + errPrinter.start(); + + int result = executor.waitFor(); + + outPrinter.join(); + errPrinter.join(); + + Boolean flag = outPrinter.hasUnexpectedDiffs(); + + //if there are differences and all the differences are known mismatch in Windows, return normal + if(result == 1 && !flag) + return 0; + + return result; + } + private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));