diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 27b353c..37895f2 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -277,6 +277,8 @@ new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"}, null), new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"}, + null), + new ReflectiveCommandHandler(this, new String[]{"delimiter"}, null) }; @@ -1404,7 +1406,7 @@ boolean isComment(String line) { // we're continuing an existing command cmd.append("\n"); cmd.append(scriptLine); - if (trimmedLine.endsWith(";")) { + if (trimmedLine.endsWith(getOpts().getDelimiter())) { // this command has terminated cmds.add(cmd.toString()); cmd = null; @@ -1425,7 +1427,7 @@ boolean isComment(String line) { // ### REVIEW: oops, somebody left the last command // unterminated; should we fix it for them or complain? // For now be nice and fix it. - cmd.append(";"); + cmd.append(getOpts().getDelimiter()); cmds.add(cmd.toString()); } } diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index f85d8a3..3ebbc9a 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -61,6 +61,7 @@ public static final char DEFAULT_DELIMITER_FOR_DSV = '|'; public static final int DEFAULT_MAX_COLUMN_WIDTH = 50; public static final int DEFAULT_INCREMENTAL_BUFFER_ROWS = 1000; + public static final String DEFAULT_DELIMITER = ";"; public static final String URL_ENV_PREFIX = "BEELINE_URL_"; @@ -116,6 +117,8 @@ private TreeSet cachedPropertyNameSet = null; + private String delimiter = DEFAULT_DELIMITER; + @Retention(RetentionPolicy.RUNTIME) public @interface Ignore { // marker annotations for functions that Reflector should ignore / pretend it does not exist @@ -659,6 +662,14 @@ public void setLastConnectedUrl(String lastConnectedUrl){ this.lastConnectedUrl = lastConnectedUrl; } + public String getDelimiter() { + return this.delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + @Ignore public static Env getEnv(){ return env; diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index d179b37..4e370d8 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -67,6 +67,7 @@ import org.apache.hive.jdbc.logs.InPlaceUpdateStream; public class Commands { + private final BeeLine beeLine; private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000; private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000; @@ -297,7 +298,7 @@ public boolean dropall(String line) { try { while (rs.next()) { cmds.add("DROP TABLE " - + rs.getString("TABLE_NAME") + ";"); + + rs.getString("TABLE_NAME") + beeLine.getOpts().getDelimiter()); } } finally { try { @@ -892,7 +893,7 @@ private boolean sourceFileInternal(File sourceFile) throws IOException { } extra = reader.readLine(); } - String[] cmds = lines.split(";"); + String[] cmds = lines.split(beeLine.getOpts().getDelimiter()); for (String c : cmds) { c = c.trim(); if (!executeInternal(c, false)) { @@ -1118,7 +1119,7 @@ public String handleMultiLineCmd(String line) throws IOException { //assumes line would never be null when this method is called private boolean isMultiLine(String line) { line = line.trim(); - if (line.endsWith(";") || beeLine.isComment(line)) { + if (line.endsWith(beeLine.getOpts().getDelimiter()) || beeLine.isComment(line)) { return false; } // handles the case like line = show tables; --test comment @@ -1189,7 +1190,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // the continuation lines! This is logged as sf.net // bug 879518. - // use multiple lines for statements not terminated by ";" + // use multiple lines for statements not terminated by the delimiter try { line = handleMultiLineCmd(line); } catch (Exception e) { @@ -1212,7 +1213,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) /** * Helper method to parse input from Beeline and convert it to a {@link List} of commands that * can be executed. This method contains logic for handling semicolons that are placed within - * quotations. It iterates through each character in the line and checks to see if it is a ;, ', + * quotations. It iterates through each character in the line and checks to see if it is the delimiter, ', * or " */ private List getCmdList(String line, boolean entireLineAsCommand) { @@ -1230,7 +1231,6 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // Index of the last seen semicolon in the given line int lastSemiColonIndex = 0; - char[] lineChars = line.toCharArray(); // Marker to track if the previous character was an escape character boolean wasPrevEscape = false; @@ -1239,43 +1239,39 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) // Iterate through the line and invoke the addCmdPart method whenever a semicolon is seen that is not inside a // quoted string - for (; index < lineChars.length; index++) { - switch (lineChars[index]) { - case '\'': - // If a single quote is seen and the index is not inside a double quoted string and the previous character - // was not an escape, then update the hasUnterminatedSingleQuote flag - if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { - hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; - } - wasPrevEscape = false; - break; - case '\"': - // If a double quote is seen and the index is not inside a single quoted string and the previous character - // was not an escape, then update the hasUnterminatedDoubleQuote flag - if (!hasUnterminatedSingleQuote && !wasPrevEscape) { - hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote; - } - wasPrevEscape = false; - break; - case ';': - // If a semicolon is seen, and the line isn't inside a quoted string, then treat - // line[lastSemiColonIndex] to line[index] as a single command - if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { - addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); - lastSemiColonIndex = index + 1; - } - wasPrevEscape = false; - break; - case '\\': - wasPrevEscape = !wasPrevEscape; - break; - default: - wasPrevEscape = false; - break; + for (; index < line.length();) { + if (line.startsWith("\'", index)) { + // If a single quote is seen and the index is not inside a double quoted string and the previous character + // was not an escape, then update the hasUnterminatedSingleQuote flag + if (!hasUnterminatedDoubleQuote && !wasPrevEscape) { + hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote; + } + wasPrevEscape = false; + index++; + } else if (line.startsWith("\"", index)) { + // If a double quote is seen and the index is not inside a single quoted string and the previous character + // was not an escape, then update the hasUnterminatedDoubleQuote flag + if (!hasUnterminatedSingleQuote && !wasPrevEscape) { + hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote; + } + wasPrevEscape = false; + index++; + } else if (line.startsWith(beeLine.getOpts().getDelimiter(), index)) { + // If a semicolon is seen, and the line isn't inside a quoted string, then treat + // line[lastSemiColonIndex] to line[index] as a single command + if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) { + addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); + lastSemiColonIndex = index + beeLine.getOpts().getDelimiter().length(); + } + wasPrevEscape = false; + index += beeLine.getOpts().getDelimiter().length(); + } else { + wasPrevEscape = line.startsWith("\\", index) && !wasPrevEscape; + index++; } } - // If the line doesn't end with a ; or if the line is empty, add the cmd part - if (lastSemiColonIndex != index || lineChars.length == 0) { + // If the line doesn't end with the delimiter or if the line is empty, add the cmd part + if (lastSemiColonIndex != index || line.length() == 0) { addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index)); } } @@ -1288,7 +1284,7 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) */ private void addCmdPart(List cmdList, StringBuilder command, String cmdpart) { if (cmdpart.endsWith("\\")) { - command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";"); + command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(beeLine.getOpts().getDelimiter()); return; } else { command.append(cmdpart); @@ -1926,4 +1922,8 @@ public boolean manual(String line) throws IOException { breader.close(); return true; } + + public boolean delimiter(String line) { + return set("set " + line); + } } diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index 7011221..ba6b4de 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -73,6 +73,7 @@ help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. help-addlocaldrivername: Add driver name that needs to be supported in the beeline client side. +help-delimiter: Sets the query delimiter, defaults to ; jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -203,6 +204,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --isolation=LEVEL set the transaction isolation level\n \ \ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ \ --maxHistoryRows=MAXHISTORYROWS The maximum number of rows to store beeline history.\n \ +\ --delimiter=DELIMITER set the query delimiter, defaults to ; \ --help display this message\n \ \n \ \ Example:\n \ diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 75f46ec..d90165b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -1082,4 +1082,46 @@ public void testBackslashInLiteral() throws Throwable { argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } + + @Test + public void testCustomDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomMultiCharDelimiter() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$$"); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } + + @Test + public void testCustomDelimiterWithMultiQuery() throws Throwable { + String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$select 'world', 'world', 'world'$"; + final String EXPECTED_PATTERN1 = "hello\thello\thello"; + final String EXPECTED_PATTERN2 = "world\tworld\tworld"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--delimiter=$"); + argList.add("--outputformat=tsv2"); + List> expectedMatches = Arrays.asList(new Tuple<>(EXPECTED_PATTERN1, true), + new Tuple<>(EXPECTED_PATTERN2, true)); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, expectedMatches); + } + + @Test + public void testCustomDelimiterBeelineCmd() throws Throwable { + String SCRIPT_TEXT = "!delimiter $\n select 'hello', 'hello', 'hello'$"; + final String EXPECTED_PATTERN = "hello\thello\thello"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv2"); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); + } }