diff --git itests/hive-unit/pom.xml itests/hive-unit/pom.xml index 6a190d1..789192b 100644 --- itests/hive-unit/pom.xml +++ itests/hive-unit/pom.xml @@ -138,12 +138,6 @@ org.apache.hive - hive-jdbc - ${project.version} - test - - - org.apache.hive hive-metastore ${project.version} tests @@ -382,12 +376,6 @@ true - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - test - - org.apache.curator curator-test ${curator.version} diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 42ef280..3e030e3 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; @@ -34,12 +35,17 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -76,6 +82,7 @@ argList.add(userName); return argList; } + /** * Start up a local Hive Server 2 for these tests */ @@ -119,7 +126,7 @@ private static void createTable() throws ClassNotFoundException, SQLException { try { stmt.execute("drop table " + tableName); } catch (Exception ex) { - fail(ex.toString()); + fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex)); } // create table @@ -150,7 +157,8 @@ public static void postTests() { * @return The stderr and stdout from running the script * @throws Throwable */ - private String testCommandLineScript(List argList, InputStream inputStream, OutStream streamType) + private static String testCommandLineScript(List argList, InputStream inputStream, + OutStream streamType) throws Throwable { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -177,7 +185,7 @@ private String testCommandLineScript(List argList, InputStream inputStre * Attempt to execute a simple script file with the -f and -i option to * BeeLine to test for presence of an expected pattern in the output (stdout * or stderr), fail if not found. Print PASSED or FAILED - * + * * @param expectedRegex * Text to look for in command output (stdout) * @param shouldMatch @@ -185,9 +193,11 @@ private String testCommandLineScript(List argList, InputStream inputStre * @throws Exception * on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, OutStream.OUT); + private void testScriptFile(String scriptText, List argList, String expectedRegex, + boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); } /** @@ -195,33 +205,40 @@ private void testScriptFile(String scriptText, String expectedRegex, * to BeeLine to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output (stdout) - * @param shouldMatch true if the pattern should be found, false if it should not * @param argList arguments * @param outType output stream type + * @param expectedRegex Text to look for in command output (stdout) + * @param shouldMatch true if the pattern should be found, false if it should not * @throws Throwable */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList, OutStream outType) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, outType); + private void testScriptFile(String scriptText, List argList, OutStream outType, + String expectedRegex, boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, outType, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); + } + + private void testScriptFile(String scriptText, List argList, OutStream streamType, + List> expectedMatches) throws Throwable { + testScriptFile(scriptText, argList, streamType, expectedMatches, + Arrays.asList(Modes.values())); } - + /** * Attempt to execute a simple script file with the -f or -i option * to BeeLine (or both) to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output/error - * @param shouldMatch true if the pattern should be found, false if it should not - * @param testScript Whether we should test -f - * @param testInit Whether we should test -i + * @param scriptText script to test the output for + * @param argList arguments to be passed to the script file to execute and produce output * @param streamType Whether match should be done against STDERR or STDOUT + * @param expectedMatches List of Tuple's defining the pattern to match and result of matching + * @param modes testing modes we have to run the script as * @throws Exception on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList, - boolean testScript, boolean testInit, OutStream streamType) throws Throwable { - + private void testScriptFile(String scriptText, List argList, + OutStream streamType, List> expectedMatches, List modes) + throws Throwable { // Put the script content in a temp file File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); System.out.println("script file is " + scriptFile.getAbsolutePath()); @@ -230,43 +247,59 @@ private void testScriptFile(String scriptText, String expectedRegex, os.print(scriptText); os.close(); - Pattern expectedPattern = Pattern.compile(".*" + expectedRegex + ".*", Pattern.DOTALL); - if (testScript) { - List copy = new ArrayList(argList); - copy.add("-f"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, null, streamType); - - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + List> patternsToBeMatched = Lists.transform(expectedMatches, + new Function, Tuple>() { + @Override + public Tuple apply(Tuple tuple) { + return new Tuple<>( + Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL), + tuple.shouldMatch + ); + } + }); + + for (Modes mode : modes) { + String output = mode.output(scriptFile, argList, streamType); + for (Tuple patternToMatch : patternsToBeMatched) { + Matcher m = patternToMatch.pattern.matcher(output); + boolean matches = m.matches(); + if (patternToMatch.shouldMatch != matches) { + //failed + fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") + + " contain " + patternToMatch.pattern.pattern()); + } } } + scriptFile.delete(); + } - // Not all scripts can be used as init scripts, so we parameterize. - // (scripts that test !connect, for eg., since -i runs after connects) - // So, we keep this optional. Most tests should leave this as true, however. - if (testInit) { - List copy = new ArrayList(argList); - copy.add("-i"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + /* + We are testing for both type of modes always so not passing that as a parameter for now + */ + enum Modes { + INIT { + @Override + String output(File scriptFile, List argList, OutStream streamType) throws Throwable { + List copy = new ArrayList<>(argList); + copy.add("-i"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); } - } - scriptFile.delete(); + }, SCRIPT { + @Override + String output(File scriptFile, List argList, OutStream streamType) throws Throwable { + List copy = new ArrayList<>(argList); + copy.add("-f"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, null, streamType); + } + }; + + abstract String output(File scriptFile, List argList, OutStream streamType) + throws Throwable; } + /** * Attempt to execute the enclosed query with the -e option to BeeLine * Test for presence of an expected pattern @@ -302,7 +335,7 @@ public void testWhitespaceBeforeCommentScriptFile() throws Throwable { final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; final String EXPECTED_PATTERN = "cannot recognize input near ''"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } /** @@ -316,7 +349,7 @@ public void testPositiveScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\n"; final String EXPECTED_PATTERN = " default "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -328,7 +361,7 @@ public void testLastLineCmdInScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\nshow tables;"; final String EXPECTED_PATTERN = " testbeelinetable1 "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -344,7 +377,7 @@ public void testBeelineHiveVariable() throws Throwable { argList.add("DUMMY_TBL=dummy"); final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -355,7 +388,7 @@ public void testBeelineHiveConfVariable() throws Throwable { final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n" + " drop table ${hiveconf:test.hive.table.name};\n"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -384,7 +417,7 @@ public void testBeelineMultiHiveVariable() throws Throwable { + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});" + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n"; final String EXPECTED_PATTERN = "dummy2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -397,7 +430,7 @@ public void testBreakOnErrorScriptFile() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n"; final String EXPECTED_PATTERN = " default "; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } @Test @@ -406,7 +439,7 @@ public void testTabInScriptFile() throws Throwable { final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;" + "\ndrop table testTabInScriptFile"; final String EXPECTED_PATTERN = "testTabInScriptFile"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -414,7 +447,7 @@ public void testBeelineShellCommand() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -426,7 +459,7 @@ public void testNullDefault() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -438,14 +471,14 @@ public void testNullNonEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } @Test public void testGetVariableValue() throws Throwable { final String SCRIPT_TEXT = "set env:TERM;"; final String EXPECTED_PATTERN = "env:TERM"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -457,13 +490,13 @@ public void testGetVariableValue() throws Throwable { @Test public void testNullEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "abc,,def"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -477,7 +510,7 @@ public void testDSVOutput() throws Throwable { argList.add("--delimiterForDSV=;"); final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -490,7 +523,7 @@ public void testTSV2Output() throws Throwable { argList.add("--outputformat=tsv2"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -503,7 +536,7 @@ public void testTSVOutput() throws Throwable { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -517,7 +550,7 @@ public void testTSV2OutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -532,7 +565,7 @@ public void testTSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -547,7 +580,7 @@ public void testCSV2OutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -562,7 +595,7 @@ public void testCSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -578,7 +611,7 @@ public void testDSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -593,7 +626,7 @@ public void testTSVOutputDeprecation() throws Throwable { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } /** @@ -607,7 +640,8 @@ public void testCSVOutputDeprecation() throws Throwable { argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, true, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true))); } /** @@ -619,10 +653,9 @@ public void testCSVOutput() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } - private String getFormatTestQuery() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; @@ -642,14 +675,14 @@ private String getFormatTestQueryForEableQuotes() { @Test public void testNullEmptyCmdArg() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "'abc','','def'"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--nullemptystring=true"); argList.add("--outputformat=csv"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -694,12 +727,12 @@ public void testNPE() throws UnsupportedEncodingException { beeLine.runCommands( new String[] {"!typeinfo"} ); String output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); beeLine.runCommands( new String[] {"!nativesql"} ); output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); System.out.println(">>> PASSED " + "testNPE" ); } @@ -709,21 +742,21 @@ public void testHiveVarSubstitution() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n"; final String EXPECTED_PATTERN = "dummy_t"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testEmbeddedBeelineConnection() throws Throwable{ String embeddedJdbcURL = Utils.URL_PREFIX+"/Default"; List argList = getBaseArgs(embeddedJdbcURL); - argList.add("--hivevar"); + argList.add("--hivevar"); argList.add("DUMMY_TBL=embedded_table"); // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + - "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; + "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; final String EXPECTED_PATTERN = "embedded_table"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -732,12 +765,19 @@ public void testEmbeddedBeelineConnection() throws Throwable{ */ @Test public void testQueryProgress() throws Throwable { - final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select count(*) from " + tableName + ";\n"; + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\n" + + "set hive.server2.logging.operation.level=execution;\n" + + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information - final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + final String EXPECTED_PATTERN = "ELAPSED TIME"; + final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>(UNEXPECTED_PATTERN, false) + ) + ); } /** @@ -757,8 +797,9 @@ public void testQueryProgressParallel() throws Throwable { "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "Number of reducers determined to be."; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, true + ); } /** @@ -771,7 +812,23 @@ public void testQueryProgressHidden() throws Throwable { "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Executing command"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, false); + } + + @Test + public void testQueryProgressWithHiveServer2ProgressBarDisabled() + throws Throwable { + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" + + "select count(*) from " + tableName + ";\n"; + // Check for part of log message as well as part of progress information + final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>("ELAPSED TIME", false)) + ); } @Test @@ -780,10 +837,10 @@ public void testMultiCommandsInOneline() throws Throwable { +"(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -804,10 +861,10 @@ public void testOneCommandInMultiLines() throws Throwable { + "(key int);show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -817,10 +874,10 @@ public void testEscapeSemiColonInQueries() throws Throwable { + " TERMINATED BY '\\n';show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -847,7 +904,7 @@ public void testEmbeddedBeelineOutputs() throws Throwable{ + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n" + "drop table embeddedBeelineOutputs;\n"; final String EXPECTED_PATTERN = "Stage-1 map ="; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } @Test @@ -855,7 +912,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{ List argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1")); final String SCRIPT_TEXT = "set var1"; final String EXPECTED_PATTERN = "var1=value1"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -865,7 +922,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{ @Test public void testBeelineConnectEnvVar() throws Throwable { final String jdbcUrl = miniHS2.getBaseJdbcURL(); - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("-u"); argList.add("blue"); argList.add("-d"); @@ -892,7 +949,9 @@ public String get(String envVar) { }; BeeLineOpts.setEnv(newEnv); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -904,11 +963,13 @@ public void testBeelineReconnect() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!close\n" + - "!reconnect\n\n\n" + - "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; + "!reconnect\n\n\n" + + "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; final String EXPECTED_PATTERN = "reconnecttest"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } @@ -921,14 +982,14 @@ public void testBeelineReconnect() throws Throwable { @Test public void testConnectionWithURLParams() throws Throwable { final String EXPECTED_PATTERN = " hivetest "; - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -937,10 +998,10 @@ public void testConnectionWithURLParams() throws Throwable { @Test public void testQueryNonEscapedSemiColon() throws Throwable { String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon " - + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; + + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; final String EXPECTED_PATTERN = " nonEscapedSemiColon "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -949,7 +1010,7 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable { final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\""; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -961,21 +1022,23 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable { @Test public void testShowDbInPrompt() throws Throwable { final String EXPECTED_PATTERN = " \\(default\\)>"; - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("--showDbInPrompt"); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testBeelineShellCommandWithoutConn() throws Throwable { - List argList = new ArrayList(); + List argList = new ArrayList<>(); final String SCRIPT_TEXT = "!sh echo hello world"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -985,11 +1048,21 @@ public void testBeelineShellCommandWithoutConn() throws Throwable { @Test public void testBeelineWithForce() throws Throwable { final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n " - + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " - + "desc new_table;\n"; + + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " + + "desc new_table;\n"; final String EXPECTED_PATTERN = "2 rows selected"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--force"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); + } + + private static class Tuple { + final K pattern; + final boolean shouldMatch; + + Tuple(K pattern, boolean shouldMatch) { + this.pattern = pattern; + this.shouldMatch = shouldMatch; + } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java index bb9a5e7..2535b10 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java @@ -2,10 +2,13 @@ import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.common.log.ProgressMonitor; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.Progress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.StringWriter; import java.util.Map; @@ -48,10 +51,17 @@ private boolean showReport(String report) { || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL; } + /* + This is used to print the progress information as pure text , a sample is as below: + Map 1: 0/1 Reducer 2: 0/1 + Map 1: 0(+1)/1 Reducer 2: 0/1 + Map 1: 1/1 Reducer 2: 0(+1)/1 + Map 1: 1/1 Reducer 2: 1/1 + */ private String getReport(Map progressMap) { StringWriter reportBuffer = new StringWriter(); - SortedSet keys = new TreeSet(progressMap.keySet()); + SortedSet keys = new TreeSet<>(progressMap.keySet()); for (String s : keys) { Progress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount(); @@ -109,7 +119,9 @@ private String getReport(Map progressMap) { * same information to beeline client when requested. */ static class LogToFileFunction extends BaseUpdateFunction { - + private static final Logger LOGGER = LoggerFactory.getLogger(LogToFileFunction.class); + private boolean hiveServer2InPlaceProgressEnabled = + SessionState.get().getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_INPLACE_PROGRESS); LogToFileFunction(TezJobMonitor monitor) { super(monitor); } @@ -121,7 +133,11 @@ public void renderProgress(ProgressMonitor progressMonitor) { @Override public void renderReport(String report) { - monitor.console.printInfo(report); + if (hiveServer2InPlaceProgressEnabled) { + LOGGER.info(report); + } else { + monitor.console.printInfo(report); + } } } diff --git service/src/java/org/apache/hive/service/cli/CLIService.java service/src/java/org/apache/hive/service/cli/CLIService.java index 5bb4ee0..689b948 100644 --- service/src/java/org/apache/hive/service/cli/CLIService.java +++ service/src/java/org/apache/hive/service/cli/CLIService.java @@ -442,8 +442,8 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP * we block for a duration determined by a step function, before we return * However, if the background operation is complete, we return immediately. */ + HiveConf conf = operation.getParentSession().getHiveConf(); if (operation.shouldRunAsync()) { - HiveConf conf = operation.getParentSession().getHiveConf(); long maxTimeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); @@ -472,7 +472,7 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP } OperationStatus opStatus = operation.getStatus(); LOG.debug(opHandle + ": getOperationStatus()"); - opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation)); + opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation, conf)); return opStatus; } @@ -482,8 +482,8 @@ public HiveConf getSessionConf(SessionHandle sessionHandle) } private static final long PROGRESS_MAX_WAIT_NS = 30 * 1000000000l; - private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation) { - if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(hiveConf) + private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation, HiveConf conf) { + if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(conf) || !OperationType.EXECUTE_STATEMENT.equals(operation.getType())) { return new JobProgressUpdate(ProgressMonitor.NULL); }