diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 06888467ec70eb1780e11969dafe2bff2c432d9d..c17ca10ebdd9209d2e394a7fb52a49b7d6ab8b9e 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -53,10 +53,11 @@ public TestLocationQueries() { * @return non-zero if it failed */ @Override - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { File logFile = new File(logDir, tname + ".out"); int failedCount = 0; + StringBuilder fileNames = new StringBuilder("Files failing the location check:"); FileReader fr = new FileReader(logFile); BufferedReader in = new BufferedReader(fr); try { @@ -69,19 +70,20 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(m.group(1)); if (!f.getName().equals(locationSubdir)) { failedCount++; + fileNames.append(f.getName()).append("\r\n"); } locationCount++; } } // we always have to find at least one location, otw the test is useless if (locationCount == 0) { - return Integer.MAX_VALUE; + return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested"); } } finally { in.close(); } - return failedCount; + return QTestProcessExecResult.create(failedCount, fileNames.toString()); } public CheckResults(String outDir, String logDir, MiniClusterType miniMr, diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 9c97c318ee89f598515d7b2e14f7f8184dc1119d..02abe53925dfc25812e80a516dc07f0cb8c773d5 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -139,12 +140,14 @@ protected void runTestHelper(String tname, String fname, String fpath, boolean e if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 3e4b3732cf49c1ac14baa58b65731f407f2b1dcd..73e5632861297384f39ae81bd43ea09e55e559d0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; + import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -92,13 +94,13 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index a735346c062684eb6069565e7354b875cde44bd7..d59b650b11df7bf5ea5eb081855eae7ad45474f0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -23,7 +23,9 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper; @@ -175,13 +177,15 @@ public void runTest(String tname, String fname, String fpath) throws Exception { failed = true; qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { failed = true; - qt.failedDiff(ecode, fname, debugHint); + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { failed = true; qt.failed(e, fname, debugHint); } finally { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 71a02bca0b416c264beb00e95d2c3ab8da30b5c1..bff81dd374c1f865f417e7b16c3d17f25218cede 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -143,12 +145,14 @@ public void runTest(String tname, String fname, String fpath) throws Exception { } } - ecode = qt.checkCompareCliDriverResults(fname, outputs); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 956a42d2711296061aa09f79596885af475ce6e6..aa0b0710d7462b9c7ae5acefc101667a4ec0df83 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -120,12 +121,12 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(ecode, fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index 6225180a0735e41f5eee2c32e5b05c674e2edbdd..8320a8017ffd5d0dd6b7c4efe1ea4ae9b8ce75b0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; @@ -101,13 +102,13 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, null); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, null); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput()); } qt.clearPostTestEffects(); - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, null); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 65b2ce71d3797823c6c14002384857cb2a61749a..438a61eb596791a9024dbe9e62a349c62aa2d92f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -123,12 +125,14 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.failed(fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 8620cde37894aa5f15a4f7850c6f62c9ac8bf6d1..26b9ce10207edce46cc08bd8e71b96fea41cd9b4 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -121,11 +123,13 @@ public void runTest(String name, String fname, String fpath) throws Exception { if (ecode != 0) { qt.failed(ecode, fname, debugHint); } - ecode = qt.checkCliDriverResults(fname); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkCliDriverResults(fname); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } - } catch (Throwable e) { + } catch (Exception e) { qt.failed(e, fname, debugHint); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java new file mode 100644 index 0000000000000000000000000000000000000000..6acc7a0cc3a6c3760e1866ad64d57d297d52e4ca --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql; + + +import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; + +/** + * Standard output and return code of a process executed during the qtests + */ +public class QTestProcessExecResult { + + private static final String TRUNCATED_OUTPUT = "Output was too long and had to be truncated..."; + private static final short MAX_OUTPUT_CHAR_LENGTH = 2000; + + private final int returnCode; + private final String standardOut; + + QTestProcessExecResult(int code, String output) { + this.returnCode = code; + this.standardOut = truncatefNeeded(output); + } + + /** + * @return executed process return code + */ + public int getReturnCode() { + return this.returnCode; + } + + /** + * @return output captured from stdout while process was executing + */ + public String getCapturedOutput() { + return this.standardOut; + } + + public static QTestProcessExecResult create(int code, String output) { + return new QTestProcessExecResult(code, output); + } + + public static QTestProcessExecResult createWithoutOutput(int code) { + return new QTestProcessExecResult(code, ""); + } + + private String truncatefNeeded(String orig) { + if (orig.length() > MAX_OUTPUT_CHAR_LENGTH) { + return orig.substring(0, MAX_OUTPUT_CHAR_LENGTH) + "\r\n" + TRUNCATED_OUTPUT; + } else { + return orig; + } + } +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0b223f42f4ce2926b104b3066944421bafa1d275..7a36b46a8cf3de916ca8208c4c8c9289b633ae5c 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -39,6 +39,7 @@ import java.io.Serializable; import java.io.StringWriter; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -67,9 +68,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; @@ -1476,7 +1478,7 @@ public void convertSequenceFileToTextFile() throws Exception { true, true); } - public int checkNegativeResults(String tname, Exception e) throws Exception { + public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1499,16 +1501,16 @@ public int checkNegativeResults(String tname, Exception e) throws Exception { outfd.write(e.getMessage()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, + QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName())); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); } - return exitVal; + return result; } - public int checkParseResults(String tname, ASTNode tree) throws Exception { + public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { String outFileExtension = getOutFileExtension(tname); @@ -1524,10 +1526,10 @@ public int checkParseResults(String tname, ASTNode tree) throws Exception { outfd.write(tree.toStringTree()); outfd.close(); - int exitVal = executeDiffCommand(outf.getPath(), expf, false, false); + QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(outf.getPath(), expf); + if (overWrite) { + overwriteResults(outf.getPath(), expf); } return exitVal; @@ -1708,7 +1710,7 @@ public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - public int checkCliDriverResults(String tname) throws Exception { + public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { assert(qMap.containsKey(tname)); String outFileExtension = getOutFileExtension(tname); @@ -1717,51 +1719,54 @@ public int checkCliDriverResults(String tname) throws Exception { File f = new File(logDir, tname + outFileExtension); maskPatterns(planMask, f.getPath()); - int exitVal = executeDiffCommand(f.getPath(), + QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); - if (exitVal != 0 && overWrite) { - exitVal = overwriteResults(f.getPath(), outFileName); + if (overWrite) { + overwriteResults(f.getPath(), outFileName); } return exitVal; } - public int checkCompareCliDriverResults(String tname, List outputs) throws Exception { + public QTestProcessExecResult checkCompareCliDriverResults(String tname, List outputs) throws Exception { assert outputs.size() > 1; maskPatterns(planMask, outputs.get(0)); for (int i = 1; i < outputs.size(); ++i) { maskPatterns(planMask, outputs.get(i)); - int ecode = executeDiffCommand( + QTestProcessExecResult result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); - if (ecode != 0) { + if (result.getReturnCode() != 0) { System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); - return ecode; + return result; } } - return 0; + return QTestProcessExecResult.createWithoutOutput(0); } - private static int overwriteResults(String inFileName, String outFileName) throws Exception { + private static void overwriteResults(String inFileName, String outFileName) throws Exception { // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) // once Hive uses JAVA 7. System.out.println("Overwriting results " + inFileName + " to " + outFileName); - return executeCmd(new String[] { + int result = executeCmd(new String[] { "cp", getQuotedString(inFileName), getQuotedString(outFileName) - }); + }).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while overwriting " + + inFileName + " with " + outFileName); } - private static int executeDiffCommand(String inFileName, + private static QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace, boolean sortResults ) throws Exception { - int result = 0; + QTestProcessExecResult result; if (sortResults) { // sort will try to open the output file in write mode on windows. We need to @@ -1774,12 +1779,9 @@ private static int executeDiffCommand(String inFileName, String inSorted = inFileName + SORT_SUFFIX; String outSorted = outFileName + SORT_SUFFIX; - result = sortFiles(inFileName, inSorted); - result |= sortFiles(outFileName, outSorted); - if (result != 0) { - System.err.println("ERROR: Could not sort files before comparing"); - return result; - } + sortFiles(inFileName, inSorted); + sortFiles(outFileName, outSorted); + inFileName = inSorted; outFileName = outSorted; } @@ -1809,27 +1811,29 @@ private static int executeDiffCommand(String inFileName, return result; } - private static int sortFiles(String in, String out) throws Exception { - return executeCmd(new String[] { + private static void sortFiles(String in, String out) throws Exception { + int result = executeCmd(new String[] { "sort", getQuotedString(in), - }, out, null); + }, out, null).getReturnCode(); + if (result != 0) + throw new IllegalStateException("Unexpected error while sorting " + in); } - private static int executeCmd(Collection args) throws Exception { + private static QTestProcessExecResult executeCmd(Collection args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(String[] args) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args) throws Exception { return executeCmd(args, null, null); } - private static int executeCmd(Collection args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(Collection args, String outFile, String errFile) throws Exception { String[] cmdArray = args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } - private static int executeCmd(String[] args, String outFile, String errFile) throws Exception { + private static QTestProcessExecResult executeCmd(String[] args, String outFile, String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); PrintStream out = outFile == null ? @@ -1841,8 +1845,11 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr Process executor = Runtime.getRuntime().exec(args); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream str = new PrintStream(bos, true); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str); outPrinter.start(); errPrinter.start(); @@ -1860,7 +1867,8 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr err.close(); } - return result; + return QTestProcessExecResult. + create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8)); } private static String getQuotedString(String str){ @@ -2036,11 +2044,18 @@ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] q qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); @@ -2087,11 +2102,18 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); - int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); - if (ecode != 0) { + QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); + if (result.getReturnCode() != 0) { failed = true; - System.err.println("Test " + qfiles[i].getName() - + " results check failed with error code " + ecode); + StringBuilder builder = new StringBuilder(); + builder.append("Test ") + .append(qfiles[i].getName()) + .append(" results check failed with error code ") + .append(result.getReturnCode()); + if (Strings.isNotEmpty(result.getCapturedOutput())) { + builder.append(" and diff value ").append(result.getCapturedOutput()); + } + System.err.println(builder.toString()); outputTestFailureHelpMessage(); } } @@ -2189,16 +2211,15 @@ public void failed(String fname, String debugHint) { public void failedDiff(int ecode, String fname, String debugHint) { String message = - "Client Execution results failed with error code = " + ecode + " while executing fname=" + + "Client Execution succeeded but contained differences " + + "(error code = " + ecode + ") after executing " + fname + (debugHint != null ? (" " + debugHint) : ""); LOG.error(message); Assert.fail(message); } - public void failed(Throwable e, String fname, String debugHint) { + public void failed(Exception e, String fname, String debugHint) { String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); System.err.println("Failed query: " + fname); System.err.flush(); Assert.fail("Unexpected exception " + @@ -2238,9 +2259,6 @@ public static void addTestsToSuiteFromQfileNames( } br.close(); } catch (Exception e) { - System.err.println("Exception: " + e.getMessage()); - e.printStackTrace(); - System.err.flush(); Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 8dba0bb9d5288240414a59f9468d3adae4dd8ef1..31f69a3615fd3d7d8e84ff0ef19994606f5d0ad5 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -21,9 +21,12 @@ import java.io.Serializable; import java.util.List; + +import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.exec.Task; @@ -106,18 +109,20 @@ public void runTest(String tname, String fname, String fpath) throws Exception { fail("Unexpected success for query: " + fname + debugHint); } catch (ParseException pe) { - int ecode = qt.checkNegativeResults(fname, pe); - if (ecode != 0) { - qt.failed(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, pe); + if (result.getReturnCode() != 0) { + qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint); } } catch (SemanticException se) { - int ecode = qt.checkNegativeResults(fname, se); - if (ecode != 0) { - qt.failedDiff(ecode, fname, debugHint); + QTestProcessExecResult result = qt.checkNegativeResults(fname, se); + if (result.getReturnCode() != 0) { + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? + debugHint : "\r\n" + result.getCapturedOutput(); + qt.failedDiff(result.getReturnCode(), fname, message); } } - catch (Throwable e) { + catch (Exception e) { qt.failed(e, fname, debugHint); }