diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index f85d8a3..3ebbc9a 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -61,6 +61,7 @@ public static final char DEFAULT_DELIMITER_FOR_DSV = '|'; public static final int DEFAULT_MAX_COLUMN_WIDTH = 50; public static final int DEFAULT_INCREMENTAL_BUFFER_ROWS = 1000; + public static final String DEFAULT_DELIMITER = ";"; public static final String URL_ENV_PREFIX = "BEELINE_URL_"; @@ -116,6 +117,8 @@ private TreeSet cachedPropertyNameSet = null; + private String delimiter = DEFAULT_DELIMITER; + @Retention(RetentionPolicy.RUNTIME) public @interface Ignore { // marker annotations for functions that Reflector should ignore / pretend it does not exist @@ -659,6 +662,14 @@ public void setLastConnectedUrl(String lastConnectedUrl){ this.lastConnectedUrl = lastConnectedUrl; } + public String getDelimiter() { + return this.delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + @Ignore public static Env getEnv(){ return env; diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 2578728..7382539 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -55,7 +55,6 @@ import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.hive.conf.VariableSubstitution; @@ -68,6 +67,7 @@ import org.apache.hive.jdbc.logs.InPlaceUpdateStream; public class Commands { + private final BeeLine beeLine; private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000; private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000; @@ -298,7 +298,7 @@ public boolean dropall(String line) { try { while (rs.next()) { cmds.add("DROP TABLE " - + rs.getString("TABLE_NAME") + ";"); + + rs.getString("TABLE_NAME") + beeLine.getOpts().getDelimiter()); } } finally { try { @@ -893,7 +893,7 @@ private boolean sourceFileInternal(File sourceFile) throws IOException { } extra = reader.readLine(); } - String[] cmds = lines.split(";"); + String[] cmds = lines.split(beeLine.getOpts().getDelimiter()); for (String c : cmds) { c = c.trim(); if (!executeInternal(c, false)) { @@ -1119,7 +1119,7 @@ public String handleMultiLineCmd(String line) throws IOException { //assumes line would never be null when this method is called private boolean isMultiLine(String line) { line = line.trim(); - if (line.endsWith(";") || beeLine.isComment(line)) { + if (line.endsWith(beeLine.getOpts().getDelimiter()) || beeLine.isComment(line)) { return false; } // handles the case like line = show tables; --test comment @@ -1190,7 +1190,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // the continuation lines! This is logged as sf.net // bug 879518. - // use multiple lines for statements not terminated by ";" + // use multiple lines for statements not terminated by the delimiter try { line = handleMultiLineCmd(line); } catch (Exception e) { @@ -1231,7 +1231,6 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // Index of the last seen semicolon in the given line int lastSemiColonIndex = 0; - char[] lineChars = line.toCharArray(); // Marker to track if the previous character was an escape character boolean wasPrevEscape = false; @@ -1240,43 +1239,34 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // Iterate through the line and invoke the addCmdPart method whenever a semicolon is seen that is not inside a // quoted string - for (; index < lineChars.length; index++) { - switch (lineChars[index]) { - case '\'': - // If a single quote is seen and the index is not inside a double quoted string and the previous character - // was not an escape, then update the hasUnterminatedSingleQuote flag - if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { - hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; - } - wasPrevEscape = false; - break; - case '\"': + for (; index < line.length(); index++) { + if (line.startsWith("\'", index)) { + // If a single quote is seen and the index is not inside a double quoted string and the previous character + // was not an escape, then update the hasUnterminatedSingleQuote flag + if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { + hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; + } + wasPrevEscape = false; + } else if (line.startsWith("\"", index)) { // If a double quote is seen and the index is not inside a single quoted string and the previous character // was not an escape, then update the hasUnterminatedDoubleQuote flag if (!hasUnterminatedSingleQuote && !wasPrevEscape) { hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote; } wasPrevEscape = false; - break; - case ';': - // If a semicolon is seen, and the line isn't inside a quoted string, then treat - // line[lastSemiColonIndex] to line[index] as a single command - if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { - addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); - lastSemiColonIndex = index + 1; - } - wasPrevEscape = false; - break; - case '\\': - wasPrevEscape = !wasPrevEscape; - break; - default: - wasPrevEscape = false; - break; - } + } else if (line.startsWith(beeLine.getOpts().getDelimiter(), index)) { + // If a semicolon is seen, and the line isn't inside a quoted string, then treat + // line[lastSemiColonIndex] to line[index] as a single command + if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { + addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); + lastSemiColonIndex = index + 1; + } + wasPrevEscape = false; + } else + wasPrevEscape = line.startsWith("\\", index) && !wasPrevEscape; } // If the line doesn't end with a ; or if the line is empty, add the cmd part - if (lastSemiColonIndex != index || lineChars.length == 0) { + if (lastSemiColonIndex != index || line.length() == 0) { addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); } } @@ -1289,7 +1279,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) */ private void addCmdPart(List cmdList, StringBuilder command, String cmdpart) { if (cmdpart.endsWith("\\")) { - command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";"); + command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(beeLine.getOpts().getDelimiter()); return; } else { command.append(cmdpart); @@ -1812,7 +1802,7 @@ public boolean run(String line) { // we're continuing an existing command cmd.append(" \n"); cmd.append(scriptLine); - if (trimmedLine.endsWith(";")) { + if (trimmedLine.endsWith(beeLine.getOpts().getDelimiter())) { // this command has terminated cmds.add(cmd.toString()); cmd = null; @@ -1833,7 +1823,7 @@ public boolean run(String line) { // ### REVIEW: oops, somebody left the last command // unterminated; should we fix it for them or complain? // For now be nice and fix it. - cmd.append(";"); + cmd.append(beeLine.getOpts().getDelimiter()); cmds.add(cmd.toString()); } } finally { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 75f46ec..a5cede6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -1082,4 +1082,37 @@ public void testBackslashInLiteral() throws Throwable { argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } + + @Test + public void testCustomDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomMultiCharDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomDelimiterWithMultiQuery() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$select 'world', 'world', 'world'$"; + final String EXPECTED_PATTERN1 = "hello\thello\thello"; + final String EXPECTED_PATTERN2 = "world\tworld\tworld"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + List> expectedMatches = Arrays.asList(new Tuple<>(EXPECTED_PATTERN1, true), + new Tuple<>(EXPECTED_PATTERN2, true)); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, expectedMatches); + } }