diff --git itests/hive-unit/pom.xml itests/hive-unit/pom.xml
index 6a190d1..789192b 100644
--- itests/hive-unit/pom.xml
+++ itests/hive-unit/pom.xml
@@ -138,12 +138,6 @@
org.apache.hive
- hive-jdbc
- ${project.version}
- test
-
-
- org.apache.hive
hive-metastore
${project.version}
tests
@@ -382,12 +376,6 @@
true
- org.apache.hadoop
- hadoop-yarn-api
- ${hadoop.version}
- test
-
-
org.apache.curator
curator-test
${curator.version}
diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
index 42ef280..3e030e3 100644
--- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
+++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
@@ -20,6 +20,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
@@ -34,12 +35,17 @@
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -76,6 +82,7 @@
argList.add(userName);
return argList;
}
+
/**
* Start up a local Hive Server 2 for these tests
*/
@@ -119,7 +126,7 @@ private static void createTable() throws ClassNotFoundException, SQLException {
try {
stmt.execute("drop table " + tableName);
} catch (Exception ex) {
- fail(ex.toString());
+ fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex));
}
// create table
@@ -150,7 +157,8 @@ public static void postTests() {
* @return The stderr and stdout from running the script
* @throws Throwable
*/
- private String testCommandLineScript(List argList, InputStream inputStream, OutStream streamType)
+ private static String testCommandLineScript(List argList, InputStream inputStream,
+ OutStream streamType)
throws Throwable {
BeeLine beeLine = new BeeLine();
ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -177,7 +185,7 @@ private String testCommandLineScript(List argList, InputStream inputStre
* Attempt to execute a simple script file with the -f and -i option to
* BeeLine to test for presence of an expected pattern in the output (stdout
* or stderr), fail if not found. Print PASSED or FAILED
- *
+ *
* @param expectedRegex
* Text to look for in command output (stdout)
* @param shouldMatch
@@ -185,9 +193,11 @@ private String testCommandLineScript(List argList, InputStream inputStre
* @throws Exception
* on command execution error
*/
- private void testScriptFile(String scriptText, String expectedRegex,
- boolean shouldMatch, List argList) throws Throwable {
- testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, OutStream.OUT);
+ private void testScriptFile(String scriptText, List argList, String expectedRegex,
+ boolean shouldMatch) throws Throwable {
+ testScriptFile(scriptText, argList, OutStream.OUT,
+ Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))
+ );
}
/**
@@ -195,33 +205,40 @@ private void testScriptFile(String scriptText, String expectedRegex,
* to BeeLine to test for presence of an expected pattern
* in the output (stdout or stderr), fail if not found.
* Print PASSED or FAILED
- * @param expectedRegex Text to look for in command output (stdout)
- * @param shouldMatch true if the pattern should be found, false if it should not
* @param argList arguments
* @param outType output stream type
+ * @param expectedRegex Text to look for in command output (stdout)
+ * @param shouldMatch true if the pattern should be found, false if it should not
* @throws Throwable
*/
- private void testScriptFile(String scriptText, String expectedRegex,
- boolean shouldMatch, List argList, OutStream outType) throws Throwable {
- testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, outType);
+ private void testScriptFile(String scriptText, List argList, OutStream outType,
+ String expectedRegex, boolean shouldMatch) throws Throwable {
+ testScriptFile(scriptText, argList, outType,
+ Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))
+ );
+ }
+
+ private void testScriptFile(String scriptText, List argList, OutStream streamType,
+ List> expectedMatches) throws Throwable {
+ testScriptFile(scriptText, argList, streamType, expectedMatches,
+ Arrays.asList(Modes.values()));
}
-
+
/**
* Attempt to execute a simple script file with the -f or -i option
* to BeeLine (or both) to test for presence of an expected pattern
* in the output (stdout or stderr), fail if not found.
* Print PASSED or FAILED
- * @param expectedRegex Text to look for in command output/error
- * @param shouldMatch true if the pattern should be found, false if it should not
- * @param testScript Whether we should test -f
- * @param testInit Whether we should test -i
+ * @param scriptText script to test the output for
+ * @param argList arguments to be passed to the script file to execute and produce output
* @param streamType Whether match should be done against STDERR or STDOUT
+ * @param expectedMatches List of Tuple's defining the pattern to match and result of matching
+ * @param modes testing modes we have to run the script as
* @throws Exception on command execution error
*/
- private void testScriptFile(String scriptText, String expectedRegex,
- boolean shouldMatch, List argList,
- boolean testScript, boolean testInit, OutStream streamType) throws Throwable {
-
+ private void testScriptFile(String scriptText, List argList,
+ OutStream streamType, List> expectedMatches, List modes)
+ throws Throwable {
// Put the script content in a temp file
File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp");
System.out.println("script file is " + scriptFile.getAbsolutePath());
@@ -230,43 +247,59 @@ private void testScriptFile(String scriptText, String expectedRegex,
os.print(scriptText);
os.close();
- Pattern expectedPattern = Pattern.compile(".*" + expectedRegex + ".*", Pattern.DOTALL);
- if (testScript) {
- List copy = new ArrayList(argList);
- copy.add("-f");
- copy.add(scriptFile.getAbsolutePath());
-
- String output = testCommandLineScript(copy, null, streamType);
-
- Matcher m = expectedPattern.matcher(output);
- boolean matches = m.matches();
- if (shouldMatch != matches) {
- //failed
- fail("Output" + output + " should" + (shouldMatch ? "" : " not") +
- " contain " + expectedRegex);
+ List> patternsToBeMatched = Lists.transform(expectedMatches,
+ new Function, Tuple>() {
+ @Override
+ public Tuple apply(Tuple tuple) {
+ return new Tuple<>(
+ Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL),
+ tuple.shouldMatch
+ );
+ }
+ });
+
+ for (Modes mode : modes) {
+ String output = mode.output(scriptFile, argList, streamType);
+ for (Tuple patternToMatch : patternsToBeMatched) {
+ Matcher m = patternToMatch.pattern.matcher(output);
+ boolean matches = m.matches();
+ if (patternToMatch.shouldMatch != matches) {
+ //failed
+ fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") +
+ " contain " + patternToMatch.pattern.pattern());
+ }
}
}
+ scriptFile.delete();
+ }
- // Not all scripts can be used as init scripts, so we parameterize.
- // (scripts that test !connect, for eg., since -i runs after connects)
- // So, we keep this optional. Most tests should leave this as true, however.
- if (testInit) {
- List copy = new ArrayList(argList);
- copy.add("-i");
- copy.add(scriptFile.getAbsolutePath());
-
- String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType);
- Matcher m = expectedPattern.matcher(output);
- boolean matches = m.matches();
- if (shouldMatch != matches) {
- //failed
- fail("Output" + output + " should" + (shouldMatch ? "" : " not") +
- " contain " + expectedRegex);
+ /*
+ We are testing for both type of modes always so not passing that as a parameter for now
+ */
+ enum Modes {
+ INIT {
+ @Override
+ String output(File scriptFile, List argList, OutStream streamType) throws Throwable {
+ List copy = new ArrayList<>(argList);
+ copy.add("-i");
+ copy.add(scriptFile.getAbsolutePath());
+ return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType);
}
- }
- scriptFile.delete();
+ }, SCRIPT {
+ @Override
+ String output(File scriptFile, List argList, OutStream streamType) throws Throwable {
+ List copy = new ArrayList<>(argList);
+ copy.add("-f");
+ copy.add(scriptFile.getAbsolutePath());
+ return testCommandLineScript(copy, null, streamType);
+ }
+ };
+
+ abstract String output(File scriptFile, List argList, OutStream streamType)
+ throws Throwable;
}
+
/**
* Attempt to execute the enclosed query with the -e option to BeeLine
* Test for presence of an expected pattern
@@ -302,7 +335,7 @@ public void testWhitespaceBeforeCommentScriptFile() throws Throwable {
final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n";
final String EXPECTED_PATTERN = "cannot recognize input near ''";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false);
}
/**
@@ -316,7 +349,7 @@ public void testPositiveScriptFile() throws Throwable {
final String SCRIPT_TEXT = "show databases;\n";
final String EXPECTED_PATTERN = " default ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -328,7 +361,7 @@ public void testLastLineCmdInScriptFile() throws Throwable {
final String SCRIPT_TEXT = "show databases;\nshow tables;";
final String EXPECTED_PATTERN = " testbeelinetable1 ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -344,7 +377,7 @@ public void testBeelineHiveVariable() throws Throwable {
argList.add("DUMMY_TBL=dummy");
final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};";
final String EXPECTED_PATTERN = "dummy";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
@Test
@@ -355,7 +388,7 @@ public void testBeelineHiveConfVariable() throws Throwable {
final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n"
+ " drop table ${hiveconf:test.hive.table.name};\n";
final String EXPECTED_PATTERN = "dummy";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -384,7 +417,7 @@ public void testBeelineMultiHiveVariable() throws Throwable {
+ "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});"
+ "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n";
final String EXPECTED_PATTERN = "dummy2";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -397,7 +430,7 @@ public void testBreakOnErrorScriptFile() throws Throwable {
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n";
final String EXPECTED_PATTERN = " default ";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false);
}
@Test
@@ -406,7 +439,7 @@ public void testTabInScriptFile() throws Throwable {
final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;"
+ "\ndrop table testTabInScriptFile";
final String EXPECTED_PATTERN = "testTabInScriptFile";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
@Test
@@ -414,7 +447,7 @@ public void testBeelineShellCommand() throws Throwable {
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt";
final String EXPECTED_PATTERN = "hello world";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -426,7 +459,7 @@ public void testNullDefault() throws Throwable {
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
"select null from " + tableName + " limit 1 ;\n";
final String EXPECTED_PATTERN = "NULL";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()));
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
}
/**
@@ -438,14 +471,14 @@ public void testNullNonEmpty() throws Throwable {
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
"!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n";
final String EXPECTED_PATTERN = "NULL";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()));
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
}
@Test
public void testGetVariableValue() throws Throwable {
final String SCRIPT_TEXT = "set env:TERM;";
final String EXPECTED_PATTERN = "env:TERM";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()));
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
}
/**
@@ -457,13 +490,13 @@ public void testGetVariableValue() throws Throwable {
@Test
public void testNullEmpty() throws Throwable {
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
- "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
+ "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
final String EXPECTED_PATTERN = "abc,,def";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
argList.add("--outputformat=csv2");
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -477,7 +510,7 @@ public void testDSVOutput() throws Throwable {
argList.add("--delimiterForDSV=;");
final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -490,7 +523,7 @@ public void testTSV2Output() throws Throwable {
argList.add("--outputformat=tsv2");
final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -503,7 +536,7 @@ public void testTSVOutput() throws Throwable {
argList.add("--outputformat=tsv");
final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -517,7 +550,7 @@ public void testTSV2OutputWithDoubleQuotes() throws Throwable {
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false");
final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
}
@@ -532,7 +565,7 @@ public void testTSVOutputWithDoubleQuotes() throws Throwable {
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
}
@@ -547,7 +580,7 @@ public void testCSV2OutputWithDoubleQuotes() throws Throwable {
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
}
@@ -562,7 +595,7 @@ public void testCSVOutputWithDoubleQuotes() throws Throwable {
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
}
@@ -578,7 +611,7 @@ public void testDSVOutputWithDoubleQuotes() throws Throwable {
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
}
@@ -593,7 +626,7 @@ public void testTSVOutputDeprecation() throws Throwable {
argList.add("--outputformat=tsv");
final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
}
/**
@@ -607,7 +640,8 @@ public void testCSVOutputDeprecation() throws Throwable {
argList.add("--outputformat=csv");
final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, true, OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR,
+ Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)));
}
/**
@@ -619,10 +653,9 @@ public void testCSVOutput() throws Throwable {
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
argList.add("--outputformat=csv");
final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
-
private String getFormatTestQuery() {
return "set hive.support.concurrency = false;\n" +
"select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n";
@@ -642,14 +675,14 @@ private String getFormatTestQueryForEableQuotes() {
@Test
public void testNullEmptyCmdArg() throws Throwable {
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
- "select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
+ "select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
final String EXPECTED_PATTERN = "'abc','','def'";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
argList.add("--nullemptystring=true");
argList.add("--outputformat=csv");
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -694,12 +727,12 @@ public void testNPE() throws UnsupportedEncodingException {
beeLine.runCommands( new String[] {"!typeinfo"} );
String output = os.toString("UTF8");
Assert.assertFalse( output.contains("java.lang.NullPointerException") );
- Assert.assertTrue( output.contains("No current connection") );
+ assertTrue(output.contains("No current connection"));
beeLine.runCommands( new String[] {"!nativesql"} );
output = os.toString("UTF8");
Assert.assertFalse( output.contains("java.lang.NullPointerException") );
- Assert.assertTrue( output.contains("No current connection") );
+ assertTrue(output.contains("No current connection"));
System.out.println(">>> PASSED " + "testNPE" );
}
@@ -709,21 +742,21 @@ public void testHiveVarSubstitution() throws Throwable {
List argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t");
final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n";
final String EXPECTED_PATTERN = "dummy_t";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
@Test
public void testEmbeddedBeelineConnection() throws Throwable{
String embeddedJdbcURL = Utils.URL_PREFIX+"/Default";
List argList = getBaseArgs(embeddedJdbcURL);
- argList.add("--hivevar");
+ argList.add("--hivevar");
argList.add("DUMMY_TBL=embedded_table");
// Set to non-zk lock manager to avoid trying to connect to zookeeper
final String SCRIPT_TEXT =
"set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" +
- "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n";
+ "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n";
final String EXPECTED_PATTERN = "embedded_table";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -732,12 +765,19 @@ public void testEmbeddedBeelineConnection() throws Throwable{
*/
@Test
public void testQueryProgress() throws Throwable {
- final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
- "select count(*) from " + tableName + ";\n";
+ final String SCRIPT_TEXT =
+ "set hive.support.concurrency = false;\n"
+ + "set hive.server2.logging.operation.level=execution;\n"
+ + "select count(*) from " + tableName + ";\n";
// Check for part of log message as well as part of progress information
- final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()),
- OutStream.ERR);
+ final String EXPECTED_PATTERN = "ELAPSED TIME";
+ final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)";
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
+ Arrays.asList(
+ new Tuple<>(EXPECTED_PATTERN, true),
+ new Tuple<>(UNEXPECTED_PATTERN, false)
+ )
+ );
}
/**
@@ -757,8 +797,9 @@ public void testQueryProgressParallel() throws Throwable {
"select count(*) from " + tableName + ";\n";
// Check for part of log message as well as part of progress information
final String EXPECTED_PATTERN = "Number of reducers determined to be.";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()),
- OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
+ EXPECTED_PATTERN, true
+ );
}
/**
@@ -771,7 +812,23 @@ public void testQueryProgressHidden() throws Throwable {
"!set silent true\n" +
"select count(*) from " + tableName + ";\n";
final String EXPECTED_PATTERN = "Executing command";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
+ EXPECTED_PATTERN, false);
+ }
+
+ @Test
+ public void testQueryProgressWithHiveServer2ProgressBarDisabled()
+ throws Throwable {
+ final String SCRIPT_TEXT =
+ "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" +
+ "select count(*) from " + tableName + ";\n";
+ // Check for part of log message as well as part of progress information
+ final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)";
+ testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
+ Arrays.asList(
+ new Tuple<>(EXPECTED_PATTERN, true),
+ new Tuple<>("ELAPSED TIME", false))
+ );
}
@Test
@@ -780,10 +837,10 @@ public void testMultiCommandsInOneline() throws Throwable {
+"(key int);show tables; --multicommands in one line";
final String EXPECTED_PATTERN = " multicmdtbl ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;";
- testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList);
+ testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
}
@Test
@@ -804,10 +861,10 @@ public void testOneCommandInMultiLines() throws Throwable {
+ "(key int);show tables; --one command in multiple lines";
final String EXPECTED_PATTERN = " multicmdtbl ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;";
- testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList);
+ testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
}
@Test
@@ -817,10 +874,10 @@ public void testEscapeSemiColonInQueries() throws Throwable {
+ " TERMINATED BY '\\n';show tables; --one command in multiple lines";
final String EXPECTED_PATTERN = " multicmdtbl ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;";
- testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList);
+ testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
}
@Test
@@ -847,7 +904,7 @@ public void testEmbeddedBeelineOutputs() throws Throwable{
+ "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n"
+ "drop table embeddedBeelineOutputs;\n";
final String EXPECTED_PATTERN = "Stage-1 map =";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
}
@Test
@@ -855,7 +912,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{
List argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1"));
final String SCRIPT_TEXT = "set var1";
final String EXPECTED_PATTERN = "var1=value1";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -865,7 +922,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{
@Test
public void testBeelineConnectEnvVar() throws Throwable {
final String jdbcUrl = miniHS2.getBaseJdbcURL();
- List argList = new ArrayList();
+ List argList = new ArrayList<>();
argList.add("-u");
argList.add("blue");
argList.add("-d");
@@ -892,7 +949,9 @@ public String get(String envVar) {
};
BeeLineOpts.setEnv(newEnv);
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
+ Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
+ Collections.singletonList(Modes.SCRIPT));
}
/**
@@ -904,11 +963,13 @@ public void testBeelineReconnect() throws Throwable {
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
final String SCRIPT_TEXT =
"!close\n" +
- "!reconnect\n\n\n" +
- "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n";
+ "!reconnect\n\n\n" +
+ "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n";
final String EXPECTED_PATTERN = "reconnecttest";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
+ Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
+ Collections.singletonList(Modes.SCRIPT));
}
@@ -921,14 +982,14 @@ public void testBeelineReconnect() throws Throwable {
@Test
public void testConnectionWithURLParams() throws Throwable {
final String EXPECTED_PATTERN = " hivetest ";
- List argList = new ArrayList();
+ List argList = new ArrayList<>();
argList.add("-d");
argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
argList.add("-u");
argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive");
String SCRIPT_TEXT = "select current_user();";
- testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -937,10 +998,10 @@ public void testConnectionWithURLParams() throws Throwable {
@Test
public void testQueryNonEscapedSemiColon() throws Throwable {
String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon "
- + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;";
+ + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;";
final String EXPECTED_PATTERN = " nonEscapedSemiColon ";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
@Test
@@ -949,7 +1010,7 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable {
final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\"";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
argList.add("--outputformat=tsv2");
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
/**
@@ -961,21 +1022,23 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable {
@Test
public void testShowDbInPrompt() throws Throwable {
final String EXPECTED_PATTERN = " \\(default\\)>";
- List argList = new ArrayList();
+ List argList = new ArrayList<>();
argList.add("--showDbInPrompt");
argList.add("-u");
argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive");
String SCRIPT_TEXT = "select current_user();";
- testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
@Test
public void testBeelineShellCommandWithoutConn() throws Throwable {
- List argList = new ArrayList();
+ List argList = new ArrayList<>();
final String SCRIPT_TEXT = "!sh echo hello world";
final String EXPECTED_PATTERN = "hello world";
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
+ Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
+ Collections.singletonList(Modes.SCRIPT));
}
/**
@@ -985,11 +1048,21 @@ public void testBeelineShellCommandWithoutConn() throws Throwable {
@Test
public void testBeelineWithForce() throws Throwable {
final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n "
- + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n "
- + "desc new_table;\n";
+ + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n "
+ + "desc new_table;\n";
final String EXPECTED_PATTERN = "2 rows selected";
List argList = getBaseArgs(miniHS2.getBaseJdbcURL());
argList.add("--force");
- testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR);
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
+ }
+
+ private static class Tuple {
+ final K pattern;
+ final boolean shouldMatch;
+
+ Tuple(K pattern, boolean shouldMatch) {
+ this.pattern = pattern;
+ this.shouldMatch = shouldMatch;
+ }
}
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
index bb9a5e7..2535b10 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
@@ -2,10 +2,13 @@
import org.apache.hadoop.hive.common.log.InPlaceUpdate;
import org.apache.hadoop.hive.common.log.ProgressMonitor;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.Progress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.StringWriter;
import java.util.Map;
@@ -48,10 +51,17 @@ private boolean showReport(String report) {
|| System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL;
}
+ /*
+ This is used to print the progress information as pure text , a sample is as below:
+ Map 1: 0/1 Reducer 2: 0/1
+ Map 1: 0(+1)/1 Reducer 2: 0/1
+ Map 1: 1/1 Reducer 2: 0(+1)/1
+ Map 1: 1/1 Reducer 2: 1/1
+ */
private String getReport(Map progressMap) {
StringWriter reportBuffer = new StringWriter();
- SortedSet keys = new TreeSet(progressMap.keySet());
+ SortedSet keys = new TreeSet<>(progressMap.keySet());
for (String s : keys) {
Progress progress = progressMap.get(s);
final int complete = progress.getSucceededTaskCount();
@@ -109,7 +119,9 @@ private String getReport(Map progressMap) {
* same information to beeline client when requested.
*/
static class LogToFileFunction extends BaseUpdateFunction {
-
+ private static final Logger LOGGER = LoggerFactory.getLogger(LogToFileFunction.class);
+ private boolean hiveServer2InPlaceProgressEnabled =
+ SessionState.get().getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_INPLACE_PROGRESS);
LogToFileFunction(TezJobMonitor monitor) {
super(monitor);
}
@@ -121,7 +133,11 @@ public void renderProgress(ProgressMonitor progressMonitor) {
@Override
public void renderReport(String report) {
- monitor.console.printInfo(report);
+ if (hiveServer2InPlaceProgressEnabled) {
+ LOGGER.info(report);
+ } else {
+ monitor.console.printInfo(report);
+ }
}
}
diff --git service/src/java/org/apache/hive/service/cli/CLIService.java service/src/java/org/apache/hive/service/cli/CLIService.java
index 5bb4ee0..689b948 100644
--- service/src/java/org/apache/hive/service/cli/CLIService.java
+++ service/src/java/org/apache/hive/service/cli/CLIService.java
@@ -442,8 +442,8 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP
* we block for a duration determined by a step function, before we return
* However, if the background operation is complete, we return immediately.
*/
+ HiveConf conf = operation.getParentSession().getHiveConf();
if (operation.shouldRunAsync()) {
- HiveConf conf = operation.getParentSession().getHiveConf();
long maxTimeout = HiveConf.getTimeVar(conf,
HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);
@@ -472,7 +472,7 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP
}
OperationStatus opStatus = operation.getStatus();
LOG.debug(opHandle + ": getOperationStatus()");
- opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation));
+ opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation, conf));
return opStatus;
}
@@ -482,8 +482,8 @@ public HiveConf getSessionConf(SessionHandle sessionHandle)
}
private static final long PROGRESS_MAX_WAIT_NS = 30 * 1000000000l;
- private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation) {
- if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(hiveConf)
+ private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation, HiveConf conf) {
+ if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(conf)
|| !OperationType.EXECUTE_STATEMENT.equals(operation.getType())) {
return new JobProgressUpdate(ProgressMonitor.NULL);
}