diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index a589f33..444b3cf 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -281,6 +281,8 @@ new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"}, null), new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"}, + null), + new ReflectiveCommandHandler(this, new String[]{"delimiter"}, null) }; @@ -1357,7 +1359,7 @@ boolean needsContinuation(String line) { return false; } - return !trimmed.endsWith(";"); + return !trimmed.endsWith(getOpts().getDelimiter()); } /** @@ -1408,7 +1410,7 @@ boolean isComment(String line) { // we're continuing an existing command cmd.append("\n"); cmd.append(scriptLine); - if (trimmedLine.endsWith(";")) { + if (trimmedLine.endsWith(getOpts().getDelimiter())) { // this command has terminated cmds.add(cmd.toString()); cmd = null; @@ -1429,7 +1431,7 @@ boolean isComment(String line) { // ### REVIEW: oops, somebody left the last command // unterminated; should we fix it for them or complain? // For now be nice and fix it. - cmd.append(";"); + cmd.append(getOpts().getDelimiter()); cmds.add(cmd.toString()); } } diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index f85d8a3..3ebbc9a 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -61,6 +61,7 @@ public static final char DEFAULT_DELIMITER_FOR_DSV = '|'; public static final int DEFAULT_MAX_COLUMN_WIDTH = 50; public static final int DEFAULT_INCREMENTAL_BUFFER_ROWS = 1000; + public static final String DEFAULT_DELIMITER = ";"; public static final String URL_ENV_PREFIX = "BEELINE_URL_"; @@ -116,6 +117,8 @@ private TreeSet cachedPropertyNameSet = null; + private String delimiter = DEFAULT_DELIMITER; + @Retention(RetentionPolicy.RUNTIME) public @interface Ignore { // marker annotations for functions that Reflector should ignore / pretend it does not exist @@ -659,6 +662,14 @@ public void setLastConnectedUrl(String lastConnectedUrl){ this.lastConnectedUrl = lastConnectedUrl; } + public String getDelimiter() { + return this.delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + @Ignore public static Env getEnv(){ return env; diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 407e018..da896a7 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -67,6 +67,7 @@ import org.apache.hive.jdbc.logs.InPlaceUpdateStream; public class Commands { + private final BeeLine beeLine; private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000; private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000; @@ -297,7 +298,7 @@ public boolean dropall(String line) { try { while (rs.next()) { cmds.add("DROP TABLE " - + rs.getString("TABLE_NAME") + ";"); + + rs.getString("TABLE_NAME") + beeLine.getOpts().getDelimiter()); } } finally { try { @@ -892,7 +893,7 @@ private boolean sourceFileInternal(File sourceFile) throws IOException { } extra = reader.readLine(); } - String[] cmds = lines.split(";"); + String[] cmds = lines.split(beeLine.getOpts().getDelimiter()); for (String c : cmds) { c = c.trim(); if (!executeInternal(c, false)) { @@ -1131,7 +1132,7 @@ public String handleMultiLineCmd(String line) throws IOException { //assumes line would never be null when this method is called private boolean isMultiLine(String line) { line = line.trim(); - if (line.endsWith(";") || beeLine.isComment(line)) { + if (line.endsWith(beeLine.getOpts().getDelimiter()) || beeLine.isComment(line)) { return false; } // handles the case like line = show tables; --test comment @@ -1202,7 +1203,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // the continuation lines! This is logged as sf.net // bug 879518. - // use multiple lines for statements not terminated by ";" + // use multiple lines for statements not terminated by the delimiter try { line = handleMultiLineCmd(line); } catch (Exception e) { @@ -1224,8 +1225,8 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) /** * Helper method to parse input from Beeline and convert it to a {@link List} of commands that - * can be executed. This method contains logic for handling semicolons that are placed within - * quotations. It iterates through each character in the line and checks to see if it is a ;, ', + * can be executed. This method contains logic for handling delimiters that are placed within + * quotations. It iterates through each character in the line and checks to see if it is the delimiter, ', * or " */ private List getCmdList(String line, boolean entireLineAsCommand) { @@ -1241,55 +1242,50 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // Marker to track if there is starting single quote without an ending double quote boolean hasUnterminatedSingleQuote = false; - // Index of the last seen semicolon in the given line - int lastSemiColonIndex = 0; - char[] lineChars = line.toCharArray(); + // Index of the last seen delimiter in the given line + int lastDelimiterIndex = 0; // Marker to track if the previous character was an escape character boolean wasPrevEscape = false; int index = 0; - // Iterate through the line and invoke the addCmdPart method whenever a semicolon is seen that is not inside a + // Iterate through the line and invoke the addCmdPart method whenever the delimiter is seen that is not inside a // quoted string - for (; index < lineChars.length; index++) { - switch (lineChars[index]) { - case '\'': - // If a single quote is seen and the index is not inside a double quoted string and the previous character - // was not an escape, then update the hasUnterminatedSingleQuote flag - if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { - hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; - } - wasPrevEscape = false; - break; - case '\"': - // If a double quote is seen and the index is not inside a single quoted string and the previous character - // was not an escape, then update the hasUnterminatedDoubleQuote flag - if (!hasUnterminatedSingleQuote && !wasPrevEscape) { - hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote; - } - wasPrevEscape = false; - break; - case ';': - // If a semicolon is seen, and the line isn't inside a quoted string, then treat - // line[lastSemiColonIndex] to line[index] as a single command - if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { - addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); - lastSemiColonIndex = index + 1; - } - wasPrevEscape = false; - break; - case '\\': - wasPrevEscape = !wasPrevEscape; - break; - default: - wasPrevEscape = false; - break; + for (; index < line.length();) { + if (line.startsWith("\'", index)) { + // If a single quote is seen and the index is not inside a double quoted string and the previous character + // was not an escape, then update the hasUnterminatedSingleQuote flag + if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { + hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; + } + wasPrevEscape = false; + index++; + } else if (line.startsWith("\"", index)) { + // If a double quote is seen and the index is not inside a single quoted string and the previous character + // was not an escape, then update the hasUnterminatedDoubleQuote flag + if (!hasUnterminatedSingleQuote && !wasPrevEscape) { + hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote; + } + wasPrevEscape = false; + index++; + } else if (line.startsWith(beeLine.getOpts().getDelimiter(), index)) { + // If the delimiter is seen, and the line isn't inside a quoted string, then treat + // line[lastDelimiterIndex] to line[index] as a single command + if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { + addCmdPart(cmdList, command, line.substring(lastDelimiterIndex, index)); + lastDelimiterIndex = index + beeLine.getOpts().getDelimiter().length(); + } + wasPrevEscape = false; + index += beeLine.getOpts().getDelimiter().length(); + } else { + wasPrevEscape = line.startsWith("\\", index) && !wasPrevEscape; + index++; } } - // If the line doesn't end with a ; or if the line is empty, add the cmd part - if (lastSemiColonIndex != index || lineChars.length == 0) { - addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); + // If the line doesn't end with the delimiter or if the line is empty, add the cmd part + if (lastDelimiterIndex != index || line.length() == 0) { + addCmdPart(cmdList, command, line.substring(lastDelimiterIndex, index)); } } return cmdList; @@ -1301,7 +1297,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) */ private void addCmdPart(List cmdList, StringBuilder command, String cmdpart) { if (cmdpart.endsWith("\\")) { - command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";"); + command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(beeLine.getOpts().getDelimiter()); return; } else { command.append(cmdpart); @@ -1949,4 +1945,8 @@ public boolean manual(String line) throws IOException { breader.close(); return true; } + + public boolean delimiter(String line) { + return set("set " + line); + } } diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index 7011221..3b8e3e6 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -73,6 +73,7 @@ help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. help-addlocaldrivername: Add driver name that needs to be supported in the beeline client side. +help-delimiter: Sets the query delimiter, defaults to ; jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -203,6 +204,8 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --isolation=LEVEL set the transaction isolation level\n \ \ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ \ --maxHistoryRows=MAXHISTORYROWS The maximum number of rows to store beeline history.\n \ +\ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \ +\ marks, slashes, and -- are not allowed; defaults to ;\n \ \ --help display this message\n \ \n \ \ Example:\n \ diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 75f46ec..d90165b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -1082,4 +1082,46 @@ public void testBackslashInLiteral() throws Throwable { argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } + + @Test + public void testCustomDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomMultiCharDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomDelimiterWithMultiQuery() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$select 'world', 'world', 'world'$"; + final String EXPECTED_PATTERN1 = "hello\thello\thello"; + final String EXPECTED_PATTERN2 = "world\tworld\tworld"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + List> expectedMatches = Arrays.asList(new Tuple<>(EXPECTED_PATTERN1, true), + new Tuple<>(EXPECTED_PATTERN2, true)); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, expectedMatches); + } + + @Test + public void testCustomDelimiterBeelineCmd() throws Throwable { + String SCRIPT_TEXT = "!delimiter $\n select 'hello', 'hello', 'hello'$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } }