diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 5364ca6..bc713fb 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -384,8 +384,11 @@ public void handle(Signal signal) { try { int lastRet = 0, ret = 0; + // we can not use "split" function directly as ";" may be quoted + List commands = splitSemiColon(line); + String command = ""; - for (String oneCmd : line.split(";")) { + for (String oneCmd : commands) { if (StringUtils.endsWith(oneCmd, "\\")) { command += StringUtils.chop(oneCmd) + ";"; @@ -415,6 +418,48 @@ public void handle(Signal signal) { } } } + + public static List splitSemiColon(String line) { + boolean insideSingleQuote = false; + boolean insideDoubleQuote = false; + int beginIndex = 0; + List ret = new ArrayList<>(); + for (int index = 0; index < line.length(); index++) { + if (line.charAt(index) == '\'') { + // take a look to see if it is escaped + if (index - 1 >= 0 && line.charAt(index - 1) == '\\') { + // it is escaped + // nothing to do + } else { + // flip the boolean variable + insideSingleQuote = !insideSingleQuote; + } + } else if (line.charAt(index) == '\"') { + // take a look to see if it is escaped + if (index - 1 >= 0 && line.charAt(index - 1) == '\\') { + // it is escaped + // nothing to do + } else { + // flip the boolean variable + insideDoubleQuote = !insideDoubleQuote; + } + } else if (line.charAt(index) == ';') { + if (insideSingleQuote || insideDoubleQuote) { + // do not split + } else { + // split, do not include ; itself + ret.add(line.substring(beginIndex, index)); + beginIndex = index + 1; + } + } else { + // nothing to do + } + } + if (beginIndex == 0) { + ret.add(line); + } + return ret; + } public int processReader(BufferedReader r) throws IOException { String line; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5bf23e7..e5797b9 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -64,6 +64,7 @@ import java.util.regex.Pattern; import com.google.common.base.Preconditions; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -1347,7 +1348,7 @@ public int executeClient(String tname) { } private int executeClientInternal(String commands) { - String [] cmds = commands.split(";"); + List cmds = CliDriver.splitSemiColon(commands); int rc = 0; String command = ""; diff --git a/ql/src/test/queries/clientpositive/specialChar.q b/ql/src/test/queries/clientpositive/specialChar.q new file mode 100644 index 0000000..81768a5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/specialChar.q @@ -0,0 +1,21 @@ +set hive.mapred.mode=nonstrict; + +CREATE TABLE ts(s varchar(550)); + +INSERT INTO ts VALUES ('Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3'); + +INSERT INTO ts VALUES ("Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"); + +INSERT INTO ts VALUES ('\''); + +INSERT INTO ts VALUES ('\"'); + +INSERT INTO ts VALUES ("Mozilla\"\'/5.0 \"(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"); + +INSERT INTO ts VALUES ("Mozilla\'\"/5.0 \'(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"); + +INSERT INTO ts VALUES ("Mozilla\\\"/5.0 ;;;;;;(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"); + +INSERT INTO ts VALUES ("Mozilla\'\\/5.0 ;;;\";;\";(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"); + +select * from ts; diff --git a/ql/src/test/results/clientpositive/specialChar.q.out b/ql/src/test/results/clientpositive/specialChar.q.out new file mode 100644 index 0000000..1e105b3 --- /dev/null +++ b/ql/src/test/results/clientpositive/specialChar.q.out @@ -0,0 +1,96 @@ +PREHOOK: query: CREATE TABLE ts(s varchar(550)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ts +POSTHOOK: query: CREATE TABLE ts(s varchar(550)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ts +PREHOOK: query: INSERT INTO ts VALUES ('Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ('Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ("Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ("Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ('\'') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ('\'') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ('\"') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ('\"') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ("Mozilla\"\'/5.0 \"(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__5 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ("Mozilla\"\'/5.0 \"(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__5 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ("Mozilla\'\"/5.0 \'(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__6 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ("Mozilla\'\"/5.0 \'(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__6 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ("Mozilla\\\"/5.0 ;;;;;;(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__7 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ("Mozilla\\\"/5.0 ;;;;;;(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__7 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO ts VALUES ("Mozilla\'\\/5.0 ;;;\";;\";(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__8 +PREHOOK: Output: default@ts +POSTHOOK: query: INSERT INTO ts VALUES ("Mozilla\'\\/5.0 ;;;\";;\";(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__8 +POSTHOOK: Output: default@ts +POSTHOOK: Lineage: ts.s EXPRESSION [(values__tmp__table__8)values__tmp__table__8.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from ts +PREHOOK: type: QUERY +PREHOOK: Input: default@ts +#### A masked pattern was here #### +POSTHOOK: query: select * from ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ts +#### A masked pattern was here #### +Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 +Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 +' +" +Mozilla"'/5.0 "(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 +Mozilla'"/5.0 '(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 +Mozilla\"/5.0 ;;;;;;(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 +Mozilla'\/5.0 ;;;";;";(iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3