diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 06888467ec70eb1780e11969dafe2bff2c432d9d..c17ca10ebdd9209d2e394a7fb52a49b7d6ab8b9e 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -53,10 +53,11 @@ public TestLocationQueries() { * @return non-zero if it failed */ @Override - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { File logFile = new File(logDir, tname + ".out"); int failedCount = 0; + StringBuilder fileNames = new StringBuilder("Files failing the location check:"); FileReader fr = new FileReader(logFile); BufferedReader in = new BufferedReader(fr); try { @@ -69,19 +70,20 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(m.group(1)); if (!f.getName().equals(locationSubdir)) { failedCount++; + fileNames.append(f.getName()).append("\r\n"); } locationCount++; } } // we always have to find at least one location, otw the test is useless if (locationCount == 0) { - return Integer.MAX_VALUE; + return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested"); } } finally { in.close(); } - return failedCount; + return QTestProcessExecResult.create(failedCount, fileNames.toString()); } public CheckResults(String outDir, String logDir, MiniClusterType miniMr, diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 9c97c318ee89f598515d7b2e14f7f8184dc1119d..02abe53925dfc25812e80a516dc07f0cb8c773d5 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -139,12 +140,14 @@ protected void runTestHelper(String tname, String fname, String fpath, boolean e if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 3e4b3732cf49c1ac14baa58b65731f407f2b1dcd..73e5632861297384f39ae81bd43ea09e55e559d0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; + import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -92,13 +94,13 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index a735346c062684eb6069565e7354b875cde44bd7..d59b650b11df7bf5ea5eb081855eae7ad45474f0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -23,7 +23,9 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper; @@ -175,13 +177,15 @@ public void runTest(String tname, String fname, String fpath) throws Exception { failed = true; qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { failed = true; - qt.failedDiff(ecode, fname, debugHint); + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { failed = true; qt.failed(e, fname, debugHint); } finally { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 71a02bca0b416c264beb00e95d2c3ab8da30b5c1..bff81dd374c1f865f417e7b16c3d17f25218cede 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -143,12 +145,14 @@ public void runTest(String tname, String fname, String fpath) throws Exception { } } - ecode = qt.checkCompareCliDriverResults(fname, outputs); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 956a42d2711296061aa09f79596885af475ce6e6..aa0b0710d7462b9c7ae5acefc101667a4ec0df83 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -120,12 +121,12 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index 6225180a0735e41f5eee2c32e5b05c674e2edbdd..8320a8017ffd5d0dd6b7c4efe1ea4ae9b8ce75b0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -101,13 +102,13 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 65b2ce71d3797823c6c14002384857cb2a61749a..438a61eb596791a9024dbe9e62a349c62aa2d92f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -123,12 +125,14 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 8620cde37894aa5f15a4f7850c6f62c9ac8bf6d1..26b9ce10207edce46cc08bd8e71b96fea41cd9b4 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -121,11 +123,13 @@ public void runTest(String name, String fname, String fpath) throws Exception { if (ecode != 0) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java index 6acc7a0cc3a6c3760e1866ad64d57d297d52e4ca..75cce14a06e141ab78dd7434faad5109f2bf79b5 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java @@ -18,12 +18,8 @@ package org.apache.hadoop.hive.ql; - -import com.google.common.base.Strings; -import org.apache.commons.lang3.StringUtils; - /** - * Standard output and return code of a process executed during the qtests + * Standard output and return code of a process executed during the qtests. */ public class QTestProcessExecResult { diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index e32ee749adf22ae30bf5b754f0e07c282a09f308..3c5c3cc11f551b5501756b073fb8f2831f032f2d 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -39,6 +39,7 @@ import java.io.Serializable; import java.io.StringWriter; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -67,9 +68,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; @@ -1466,10 +1468,6 @@ public void convertSequenceFileToTextFile() throws Exception { // Create an instance of hive in order to create the tables testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); db = Hive.get(conf); - // Create dest4 to replace dest4_sequencefile - LinkedList cols = new LinkedList(); - cols.add("key"); - cols.add("value"); // Move all data from dest4_sequencefile to dest4 drv @@ -1480,7 +1478,7 @@ public void convertSequenceFileToTextFile() throws Exception { true, true); } - public int checkNegativeResults(String tname, Exception e) throws Exception { + public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1503,16 +1501,17 @@ public int checkNegativeResults(String tname, Exception e) throws Exception { outfd.write(e.getMessage()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, + QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName())); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); + return QTestProcessExecResult.createWithoutOutput(0); } - return exitVal; + return result; } - public int checkParseResults(String tname, ASTNode tree) throws Exception { + public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { String outFileExtension = getOutFileExtension(tname); @@ -1528,10 +1527,11 @@ public int checkParseResults(String tname, ASTNode tree) throws Exception { outfd.write(tree.toStringTree()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, false); + QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); + return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; @@ -1712,7 +1712,7 @@ public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { assert(qMap.containsKey(tname)); String outFileExtension = getOutFileExtension(tname); @@ -1721,69 +1721,71 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(logDir, tname + outFileExtension); maskPatterns(planMask, f.getPath()); - int exitVal = executeDiffCommand(f.getPath(), + QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(f.getPath(), outFileName); + if (overWrite) { + overwriteResults(f.getPath(), outFileName); + return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; } - public int checkCompareCliDriverResults(String tname, List outputs) throws Exception { + public QTestProcessExecResult checkCompareCliDriverResults(String tname, List outputs) + throws Exception { assert outputs.size() > 1; maskPatterns(planMask, outputs.get(0)); for (int i = 1; i < outputs.size(); ++i) { maskPatterns(planMask, outputs.get(i)); - int ecode = executeDiffCommand( + QTestProcessExecResult result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); - if (ecode != 0) { + if (result.getReturnCode() != 0) { System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); - return ecode; + return result; } } - return 0; + return QTestProcessExecResult.createWithoutOutput(0); } - private static int overwriteResults(String inFileName, String outFileName) throws Exception { + private static void overwriteResults(String inFileName, String outFileName) throws Exception { // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) // once Hive uses JAVA 7. System.out.println("Overwriting results " + inFileName + " to " + outFileName); - return executeCmd(new String[] { + int result = executeCmd(new String[]{ "cp", getQuotedString(inFileName), getQuotedString(outFileName) - }); + }).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while overwriting " + + inFileName + " with " + outFileName); } - private static int executeDiffCommand(String inFileName, + private static QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace, boolean sortResults ) throws Exception { - int result = 0; + QTestProcessExecResult result; if (sortResults) { // sort will try to open the output file in write mode on windows. We need to // close it first. SessionState ss = SessionState.get(); if (ss != null && ss.out != null && ss.out != System.out) { - ss.out.close(); + ss.out.close(); } String inSorted = inFileName + SORT_SUFFIX; String outSorted = outFileName + SORT_SUFFIX; - result = sortFiles(inFileName, inSorted); - result |= sortFiles(outFileName, outSorted); - if (result != 0) { - System.err.println("ERROR: Could not sort files before comparing"); - return result; - } + sortFiles(inFileName, inSorted); + sortFiles(outFileName, outSorted); + inFileName = inSorted; outFileName = outSorted; } @@ -1813,40 +1815,47 @@ private static int executeDiffCommand(String inFileName, return result; } - private static int sortFiles(String in, String out) throws Exception { - return executeCmd(new String[] { + private static void sortFiles(String in, String out) throws Exception { + int result = executeCmd(new String[]{ "sort", getQuotedString(in), - }, out, null); + }, out, null).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while sorting " + in); } - private static int executeCmd(Collection args) throws Exception { + private static QTestProcessExecResult executeCmd(Collection args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(String[] args) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(Collection args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(Collection args, String outFile, + String errFile) throws Exception { String[] cmdArray = args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } - private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args, String outFile, + String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); PrintStream out = outFile == null ? SessionState.getConsole().getChildOutStream() : - new PrintStream(new FileOutputStream(outFile), true); + new PrintStream(new FileOutputStream(outFile), true, "UTF-8"); PrintStream err = errFile == null ? SessionState.getConsole().getChildErrStream() : - new PrintStream(new FileOutputStream(errFile), true); + new PrintStream(new FileOutputStream(errFile), true, "UTF-8"); Process executor = Runtime.getRuntime().exec(args); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream str = new PrintStream(bos, true, "UTF-8"); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str); outPrinter.start(); errPrinter.start(); @@ -1864,7 +1873,8 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr err.close(); } - return result; + return QTestProcessExecResult. + create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8)); } private static String getQuotedString(String str){ @@ -2040,11 +2050,18 @@ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] q qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); @@ -2091,11 +2108,18 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } } @@ -2193,16 +2217,15 @@ public void failed(String fname, String debugHint) { public void failedDiff(int ecode, String fname, String debugHint) { String message = - "Client Execution results failed with error code = " + ecode + " while executing fname=" + + "Client Execution succeeded but contained differences " + + "(error code = " + ecode + ") after executing " + fname + (debugHint != null ? (" " + debugHint) : ""); LOG.error(message); Assert.fail(message); } - public void failed(Throwable e, String fname, String debugHint) { + public void failed(Exception e, String fname, String debugHint) { String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); System.err.println("Failed query: " + fname); System.err.flush(); Assert.fail("Unexpected exception " + @@ -2242,9 +2265,6 @@ public static void addTestsToSuiteFromQfileNames( } br.close(); } catch (Exception e) { - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); - System.err.flush(); Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } @@ -2268,7 +2288,9 @@ public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf c String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/"; // Setup the table column stats - BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); + BufferedReader br = new BufferedReader( + new FileReader( + new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); String command; s.execute("DROP TABLE APP.TABLE_PARAMS"); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 8dba0bb9d5288240414a59f9468d3adae4dd8ef1..31f69a3615fd3d7d8e84ff0ef19994606f5d0ad5 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -21,9 +21,12 @@ import java.io.Serializable; import java.util.List; + +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.exec.Task; @@ -106,18 +109,20 @@ public void runTest(String tname, String fname, String fpath) throws Exception { fail("Unexpected success for query: " + fname + debugHint); } catch (ParseException pe) { - int ecode = qt.checkNegativeResults(fname, pe); - if (ecode != 0) { - qt.failed(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, pe); + if (result.getReturnCode() != 0) { + qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint); } } catch (SemanticException se) { - int ecode = qt.checkNegativeResults(fname, se); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, se); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); }