diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 0688846..91c54ac 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -24,6 +24,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; /** @@ -53,10 +54,11 @@ public TestLocationQueries() { * @return non-zero if it failed */ @Override - public int checkCliDriverResults(String tname) throws Exception { + public Pair checkCliDriverResults(String tname) throws Exception { File logFile = new File(logDir, tname + ".out"); int failedCount = 0; + StringBuilder fileNames = new StringBuilder("Files failing the location check:"); FileReader fr = new FileReader(logFile); BufferedReader in = new BufferedReader(fr); try { @@ -69,19 +71,20 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(m.group(1)); if (!f.getName().equals(locationSubdir)) { failedCount++; + fileNames.append(f.getName()).append("\r\n"); } locationCount++; } } // we always have to find at least one location, otw the test is useless if (locationCount == 0) { - return Integer.MAX_VALUE; + return Pair.of(Integer.MAX_VALUE, "0 locations tested"); } } finally { in.close(); } - return failedCount; + return Pair.of(failedCount, fileNames.toString()); } public CheckResults(String outDir, String logDir, MiniClusterType miniMr, diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 9c97c31..4f16628 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -25,6 +25,7 @@ import java.util.Calendar; import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveVariableSource; @@ -139,12 +140,13 @@ protected void runTestHelper(String tname, String fname, String fpath, boolean e if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 3e4b373..4846628 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; + +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -92,9 +94,9 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + qt.failedDiff(result.getLeft(), fname, result.getRight()); } qt.clearPostTestEffects(); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index a735346..f0ccdb7 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -23,6 +23,8 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -175,13 +177,14 @@ public void runTest(String tname, String fname, String fpath) throws Exception { failed = true; qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { failed = true; - qt.failedDiff(ecode, fname, debugHint); + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { failed = true; qt.failed(e, fname, debugHint); } finally { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 71a02bc..4d64b22 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; +import com.google.common.base.Strings; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -143,9 +145,10 @@ public void runTest(String tname, String fname, String fpath) throws Exception { } } - ecode = qt.checkCompareCliDriverResults(fname, outputs); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + Pair result = qt.checkCompareCliDriverResults(fname, outputs); + if (result.getLeft() != 0) { + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } catch (Throwable e) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 956a42d..de8a0e3 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -120,12 +121,12 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + qt.failedDiff(result.getLeft(), fname, result.getRight()); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index 6225180..6ceed10 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -101,13 +102,13 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + qt.failedDiff(result.getLeft(), fname, result.getRight()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 65b2ce7..7c39c69 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -123,9 +125,10 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } catch (Throwable e) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 8620cde..89aae48 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -121,9 +123,10 @@ public void runTest(String name, String fname, String fpath) throws Exception { if (ecode != 0) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + Pair result = qt.checkCliDriverResults(fname); + if (result.getLeft() != 0) { + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } catch (Throwable e) { qt.failed(e, fname, debugHint); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 8b383c7..3505b27 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -39,6 +39,7 @@ import java.io.Serializable; import java.io.StringWriter; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -66,6 +67,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -1511,7 +1513,7 @@ public void convertSequenceFileToTextFile() throws Exception { true, true); } - public int checkNegativeResults(String tname, Exception e) throws Exception { + public Pair checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1534,16 +1536,16 @@ public int checkNegativeResults(String tname, Exception e) throws Exception { outfd.write(e.getMessage()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, + Pair result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName())); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); } - return exitVal; + return result; } - public int checkParseResults(String tname, ASTNode tree) throws Exception { + public Pair checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { String outFileExtension = getOutFileExtension(tname); @@ -1559,10 +1561,10 @@ public int checkParseResults(String tname, ASTNode tree) throws Exception { outfd.write(tree.toStringTree()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, false); + Pair exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); } return exitVal; @@ -1742,7 +1744,7 @@ public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - public int checkCliDriverResults(String tname) throws Exception { + public Pair checkCliDriverResults(String tname) throws Exception { assert(qMap.containsKey(tname)); String outFileExtension = getOutFileExtension(tname); @@ -1751,51 +1753,54 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(logDir, tname + outFileExtension); maskPatterns(planMask, f.getPath()); - int exitVal = executeDiffCommand(f.getPath(), + Pair exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(f.getPath(), outFileName); + if (overWrite) { + overwriteResults(f.getPath(), outFileName); } return exitVal; } - public int checkCompareCliDriverResults(String tname, List outputs) throws Exception { + public Pair checkCompareCliDriverResults(String tname, List outputs) throws Exception { assert outputs.size() > 1; maskPatterns(planMask, outputs.get(0)); for (int i = 1; i < outputs.size(); ++i) { maskPatterns(planMask, outputs.get(i)); - int ecode = executeDiffCommand( + Pair result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); - if (ecode != 0) { + if (result.getLeft() != 0) { System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); - return ecode; + return result; } } - return 0; + return Pair.of(0, ""); } - private static int overwriteResults(String inFileName, String outFileName) throws Exception { + private static void overwriteResults(String inFileName, String outFileName) throws Exception { // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) // once Hive uses JAVA 7. System.out.println("Overwriting results " + inFileName + " to " + outFileName); - return executeCmd(new String[] { + int result = executeCmd(new String[] { "cp", getQuotedString(inFileName), getQuotedString(outFileName) - }); + }).getLeft(); + if (result != 0) + throw new IllegalStateException("Unexpected error while overwriting " + + inFileName + " with " + outFileName); } - private static int executeDiffCommand(String inFileName, + private static Pair executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace, boolean sortResults ) throws Exception { - int result = 0; + Pair result; if (sortResults) { // sort will try to open the output file in write mode on windows. We need to @@ -1808,12 +1813,9 @@ private static int executeDiffCommand(String inFileName, String inSorted = inFileName + SORT_SUFFIX; String outSorted = outFileName + SORT_SUFFIX; - result = sortFiles(inFileName, inSorted); - result |= sortFiles(outFileName, outSorted); - if (result != 0) { - System.err.println("ERROR: Could not sort files before comparing"); - return result; - } + sortFiles(inFileName, inSorted); + sortFiles(outFileName, outSorted); + inFileName = inSorted; outFileName = outSorted; } @@ -1852,27 +1854,29 @@ private static int executeDiffCommand(String inFileName, return result; } - private static int sortFiles(String in, String out) throws Exception { - return executeCmd(new String[] { + private static void sortFiles(String in, String out) throws Exception { + int result = executeCmd(new String[] { "sort", getQuotedString(in), - }, out, null); + }, out, null).getLeft(); + if (result != 0) + throw new IllegalStateException("Unexpected error while sorting " + in); } - private static int executeCmd(Collection args) throws Exception { + private static Pair executeCmd(Collection args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(String[] args) throws Exception { + private static Pair executeCmd(String[] args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(Collection args, String outFile, String errFile) throws Exception { + private static Pair executeCmd(Collection args, String outFile, String errFile) throws Exception { String[] cmdArray = args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } - private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { + private static Pair executeCmd(String[] args, String outFile, String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); PrintStream out = outFile == null ? @@ -1884,8 +1888,11 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr Process executor = Runtime.getRuntime().exec(args); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream str = new PrintStream(bos, true); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str); outPrinter.start(); errPrinter.start(); @@ -1903,7 +1910,7 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr err.close(); } - return result; + return Pair.of(result, new String(bos.toByteArray(), StandardCharsets.UTF_8)); } private static String getQuotedString(String str){ @@ -2079,11 +2086,18 @@ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] q qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + Pair result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getLeft() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getLeft()); + if (Strings.isNotEmpty(result.getRight())) { + builder.append(" and diff value ").append(result.getRight()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); @@ -2130,11 +2144,18 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + Pair result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getLeft() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getLeft()); + if (Strings.isNotEmpty(result.getRight())) { + builder.append(" and diff value ").append(result.getRight()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 8dba0bb..64cbdbb 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -21,6 +21,9 @@ import java.io.Serializable; import java.util.List; + +import com.google.common.base.Strings; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; @@ -106,15 +109,16 @@ public void runTest(String tname, String fname, String fpath) throws Exception { fail("Unexpected success for query: " + fname + debugHint); } catch (ParseException pe) { - int ecode = qt.checkNegativeResults(fname, pe); - if (ecode != 0) { - qt.failed(ecode, fname, debugHint); + Pair result = qt.checkNegativeResults(fname, pe); + if (result.getLeft() != 0) { + qt.failed(result.getLeft(), fname, result.getRight() + "\r\n" + debugHint); } } catch (SemanticException se) { - int ecode = qt.checkNegativeResults(fname, se); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + Pair result = qt.checkNegativeResults(fname, se); + if (result.getLeft() != 0) { + String message = Strings.isNullOrEmpty(result.getRight()) ? debugHint : "\r\n" + result.getRight(); + qt.failedDiff(result.getLeft(), fname, message); } } catch (Throwable e) {