diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 99db643..6a3ad42 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -985,7 +985,7 @@ private boolean executeInternal(String sql, boolean call) { logThread.start(); if (stmnt instanceof HiveStatement) { ((HiveStatement) stmnt).setInPlaceUpdateStream( - new BeelineInPlaceUpdateStream(beeLine.getOutputStream()) + new BeelineInPlaceUpdateStream(beeLine.getErrorStream()) ); } hasResults = stmnt.execute(sql); diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index e641253..71f9640 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -222,6 +222,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: + // Change the engine to tez + hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez"); // TODO: This should be making use of confDir to load configs setup for Tez, etc. mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, false); break; diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 9e99a91..76506df 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -29,6 +29,7 @@ import java.io.PrintStream; import java.io.StringBufferInputStream; import java.io.UnsupportedEncodingException; +import java.net.URL; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -36,11 +37,16 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -51,11 +57,17 @@ * */ public class TestBeeLineWithArgs { + private enum OUTSTREAM { + ERR, OUT + }; + // Default location of HiveServer2 private static final String tableName = "TestBeelineTable1"; private static final String tableComment = "Test table comment"; - + private static String dataFileDir; + private static Path kvDataFilePath; private static MiniHS2 miniHS2; + private static final String userName = System.getProperty("user.name"); private List getBaseArgs(String jdbcUrl) { List argList = new ArrayList(8); @@ -63,6 +75,8 @@ argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(jdbcUrl); + argList.add("-n"); + argList.add(userName); return argList; } /** @@ -71,11 +85,15 @@ @BeforeClass public static void preTests() throws Exception { HiveConf hiveConf = new HiveConf(); - // Set to non-zk lock manager to prevent HS2 from trying to connect - hiveConf.setVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); + hiveConf.setVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER, + "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, false); - miniHS2 = new MiniHS2(hiveConf); - miniHS2.start(new HashMap()); + hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); + miniHS2 = new MiniHS2(hiveConf, MiniClusterType.TEZ); + + Map confOverlay = new HashMap(); + miniHS2.start(confOverlay); + createTable(); } @@ -86,7 +104,8 @@ public static void preTests() throws Exception { */ private static void createTable() throws ClassNotFoundException, SQLException { Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); - Connection con = DriverManager.getConnection(miniHS2.getBaseJdbcURL(),"", ""); + Connection con = DriverManager.getConnection(miniHS2.getBaseJdbcURL(), + userName , ""); assertNotNull("Connection is null", con); assertFalse("Connection should not be closed", con.isClosed()); @@ -128,16 +147,27 @@ public static void postTests() { /** * Execute a script with "beeline -f or -i" - * + * @param argList List of arguments for beeline + * @param inputStream input stream if any + * @param streamType if output from STDERR or STDOUT needs to be returned * @return The stderr and stdout from running the script + * @throws Throwable */ - private String testCommandLineScript(List argList, InputStream inputStream) + private String testCommandLineScript(List argList, InputStream inputStream, OUTSTREAM streamType) throws Throwable { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream beelineOutputStream = new PrintStream(os); - beeLine.setOutputStream(beelineOutputStream); - beeLine.setErrorStream(beelineOutputStream); + switch (streamType) { + case OUT: + beeLine.setOutputStream(beelineOutputStream); + break; + case ERR: + beeLine.setErrorStream(beelineOutputStream); + break; + default: + throw new RuntimeException("Unexpected outstream type " + streamType); + } String[] args = argList.toArray(new String[argList.size()]); beeLine.begin(args, inputStream); String output = os.toString("UTF8"); @@ -147,33 +177,53 @@ private String testCommandLineScript(List argList, InputStream inputStre } /** + * Attempt to execute a simple script file with the -f and -i option to + * BeeLine to test for presence of an expected pattern in the output (stdout + * or stderr), fail if not found. Print PASSED or FAILED + * + * @param expectedRegex + * Text to look for in command output (stdout) + * @param shouldMatch + * true if the pattern should be found, false if it should not + * @throws Exception + * on command execution error + */ + private void testScriptFile(String scriptText, String expectedRegex, + boolean shouldMatch, List argList) throws Throwable { + testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, OUTSTREAM.OUT); + } + + /** * Attempt to execute a simple script file with the -f and -i option * to BeeLine to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedPattern Text to look for in command output/error + * @param expectedRegex Text to look for in command output (stdout) * @param shouldMatch true if the pattern should be found, false if it should not - * @throws Exception on command execution error + * @param argList arguments + * @param outType output stream type + * @throws Throwable */ - private void testScriptFile(String scriptText, String expectedPattern, - boolean shouldMatch, List argList) throws Throwable { - testScriptFile(scriptText, expectedPattern, shouldMatch, argList, true, true); + private void testScriptFile(String scriptText, String expectedRegex, + boolean shouldMatch, List argList, OUTSTREAM outType) throws Throwable { + testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, outType); } - + /** * Attempt to execute a simple script file with the -f or -i option * to BeeLine (or both) to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedPattern Text to look for in command output/error + * @param expectedRegex Text to look for in command output/error * @param shouldMatch true if the pattern should be found, false if it should not * @param testScript Whether we should test -f * @param testInit Whether we should test -i + * @param streamType Whether match should be done against STDERR or STDOUT * @throws Exception on command execution error */ - private void testScriptFile(String scriptText, String expectedPattern, + private void testScriptFile(String scriptText, String expectedRegex, boolean shouldMatch, List argList, - boolean testScript, boolean testInit) throws Throwable { + boolean testScript, boolean testInit, OUTSTREAM streamType) throws Throwable { // Put the script content in a temp file File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); @@ -183,17 +233,20 @@ private void testScriptFile(String scriptText, String expectedPattern, os.print(scriptText); os.close(); + Pattern expectedPattern = Pattern.compile(".*" + expectedRegex + ".*", Pattern.DOTALL); if (testScript) { List copy = new ArrayList(argList); copy.add("-f"); copy.add(scriptFile.getAbsolutePath()); - String output = testCommandLineScript(copy, null); - boolean matches = output.contains(expectedPattern); + String output = testCommandLineScript(copy, null, streamType); + + Matcher m = expectedPattern.matcher(output); + boolean matches = m.matches(); if (shouldMatch != matches) { //failed fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedPattern); + " contain " + expectedRegex); } } @@ -205,12 +258,13 @@ private void testScriptFile(String scriptText, String expectedPattern, copy.add("-i"); copy.add(scriptFile.getAbsolutePath()); - String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n")); - boolean matches = output.contains(expectedPattern); + String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); + Matcher m = expectedPattern.matcher(output); + boolean matches = m.matches(); if (shouldMatch != matches) { //failed fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedPattern); + " contain " + expectedRegex); } } scriptFile.delete(); @@ -225,14 +279,15 @@ private void testScriptFile(String scriptText, String expectedPattern, * @param shouldMatch true if the pattern should be found, false if it should not * @throws Exception on command execution error */ + private void testCommandEnclosedQuery(String enclosedQuery, String expectedPattern, - boolean shouldMatch, List argList) throws Throwable { + boolean shouldMatch, List argList, OUTSTREAM out) throws Throwable { List copy = new ArrayList(argList); copy.add("-e"); copy.add(enclosedQuery); - String output = testCommandLineScript(copy, null); + String output = testCommandLineScript(copy, null, out); boolean matches = output.contains(expectedPattern); if (shouldMatch != matches) { //failed @@ -290,7 +345,7 @@ public void testBeelineHiveVariable() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hivevar"); argList.add("DUMMY_TBL=dummy"); - final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n"; + final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};"; final String EXPECTED_PATTERN = "dummy"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -300,7 +355,8 @@ public void testBeelineHiveConfVariable() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hiveconf"); argList.add("test.hive.table.name=dummy"); - final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n"; + final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n" + + " drop table ${hiveconf:test.hive.table.name};\n"; final String EXPECTED_PATTERN = "dummy"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -327,7 +383,9 @@ public void testBeelineMultiHiveVariable() throws Throwable { argList.add("--hiveconf"); argList.add("COLUMN_TYPE=int"); - final String SCRIPT_TEXT = "${COMMAND} ${OBJECT} ${TABLE_NAME} (${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});\nshow tables;\n"; + final String SCRIPT_TEXT = "${COMMAND} ${OBJECT} ${TABLE_NAME} " + + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});" + + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n"; final String EXPECTED_PATTERN = "dummy2"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -537,7 +595,7 @@ public void testTSVOutputDeprecation() throws Throwable { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OUTSTREAM.ERR); } /** @@ -551,7 +609,7 @@ public void testCSVOutputDeprecation() throws Throwable { argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, true, OUTSTREAM.ERR); } /** @@ -612,7 +670,7 @@ public void testNegativeScriptFile() throws Throwable { argList.add(scriptFile.getAbsolutePath()); try { - String output = testCommandLineScript(argList, null); + String output = testCommandLineScript(argList, null, OUTSTREAM.OUT); if (output.contains(EXPECTED_PATTERN)) { fail("Output: " + output + " Negative pattern: " + EXPECTED_PATTERN); } @@ -651,7 +709,7 @@ public void testNPE() throws UnsupportedEncodingException { @Test public void testHiveVarSubstitution() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); - final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\n"; + final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n"; final String EXPECTED_PATTERN = "dummy_t"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -665,7 +723,7 @@ public void testEmbeddedBeelineConnection() throws Throwable{ // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + - "create table ${DUMMY_TBL} (d int);\nshow tables;\n"; + "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; final String EXPECTED_PATTERN = "embedded_table"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -678,8 +736,10 @@ public void testEmbeddedBeelineConnection() throws Throwable{ public void testQueryProgress() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select count(*) from " + tableName + ";\n"; - final String EXPECTED_PATTERN = "number of splits"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + // Check for part of log message as well as part of progress information + final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), + OUTSTREAM.ERR); } /** @@ -692,8 +752,10 @@ public void testQueryProgressParallel() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "set hive.exec.parallel = true;\n" + "select count(*) from " + tableName + ";\n"; - final String EXPECTED_PATTERN = "number of splits"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + // Check for part of log message as well as part of progress information + final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), + OUTSTREAM.ERR); } /** @@ -706,7 +768,7 @@ public void testQueryProgressHidden() throws Throwable { "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Executing command"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL()), OUTSTREAM.ERR); } @Test @@ -727,10 +789,10 @@ public void testMultiCommandsInOneEnclosedQuery() throws Throwable { +"(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList); + testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OUTSTREAM.OUT); final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;"; - testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OUTSTREAM.OUT); } @Test @@ -765,10 +827,10 @@ public void testEscapeSemiColonInEnclosedQuery() throws Throwable { + " TERMINATED BY '\\n';show tables;"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList); + testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OUTSTREAM.OUT); final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;"; - testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OUTSTREAM.OUT); } @Test @@ -781,7 +843,7 @@ public void testEmbeddedBeelineOutputs() throws Throwable{ + "create table if not exists embeddedBeelineOutputs(d int);\n" + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n"; final String EXPECTED_PATTERN = "Stage-1 map ="; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OUTSTREAM.ERR); } @Test @@ -826,7 +888,7 @@ public String get(String envVar) { }; BeeLineOpts.setEnv(newEnv); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OUTSTREAM.OUT); } /** @@ -842,7 +904,7 @@ public void testBeelineReconnect() throws Throwable { "create table reconnecttest (d int);\nshow tables;\n"; final String EXPECTED_PATTERN = "reconnecttest"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OUTSTREAM.OUT); } @@ -909,7 +971,7 @@ public void testBeelineShellCommandWithoutConn() throws Throwable { List argList = new ArrayList(); final String SCRIPT_TEXT = "!sh echo hello world"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList,true,false); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OUTSTREAM.OUT); } /** @@ -924,6 +986,6 @@ public void testBeelineWithForce() throws Throwable { final String EXPECTED_PATTERN = "2 rows selected"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--force"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OUTSTREAM.ERR); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java index e98406d..388486d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java @@ -58,8 +58,6 @@ public static void setUpBeforeClass() throws Exception { }; hiveConf = new HiveConf(); hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); - // Change the engine to tez - hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez"); // Set tez execution summary to false. hiveConf.setBoolVar(ConfVars.TEZ_EXEC_SUMMARY, false); miniHS2 = new MiniHS2(hiveConf, MiniClusterType.TEZ);