diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index f448b16..cad3741 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -144,7 +144,7 @@ fromSource @init { gParent.pushMsg("from source", state); } @after { gParent.popMsg(state); } : - ((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource) (lateralView^)* + ((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource | virtualTableSource) (lateralView^)* ; tableBucketSample @@ -256,3 +256,49 @@ searchCondition ; //----------------------------------------------------------------------------------- + +//-------- Row Constructor ---------------------------------------------------------- +//in support of SELECT * FROM VALUES((1,2,3),(4,5,6),...) and +// INSERT INTO (col1,col2,...) VALUES((...),(...),...) +// INSERT INTO
(col1,col2,...) SELECT * FROM VALUES((1,2,3),(4,5,6),...) as Foo(a,b,c) +//todo: can we have valuesClause automatically produce valuesTable when it sees valueRow +// both of these seem OK even when referenced from regularBody in HiveParser.g +valueRowConstructor + : + LPAREN atomExpression (COMMA atomExpression)* RPAREN -> ^(TOK_VALUE_ROW atomExpression+) + ; + +valuesTableConstructor + : + LPAREN valueRowConstructor (COMMA valueRowConstructor)* RPAREN -> ^(TOK_VALUES_TABLE valueRowConstructor+) + ; + +/* +The parse tree for VALUES(1,2) and VALUES((1,2)) is identical. +The former is just syntactic sugar for the end user +*/ +valuesClause + : + KW_VALUES valuesTableConstructor -> valuesTableConstructor + | + KW_VALUES valueRowConstructor -> ^(TOK_VALUES_TABLE valueRowConstructor) + ; + +/* +This represents a clause like this: +VALUES((1,2),(2,3)) as VirtTable(col1,col2) +*/ +virtualTableSource + : + valuesClause tableNameColList -> ^(TOK_VIRTUAL_TABLE tableNameColList valuesClause) + ; +/* +e.g. as VirtTable(col1,col2) +Note that we only want literals as column names +*/ +tableNameColList + : + KW_AS? identifier LPAREN identifier (COMMA identifier)* RPAREN -> ^(TOK_VIRTUAL_TABREF ^(TOK_TABNAME identifier) ^(TOK_COL_NAME identifier+)) + ; + +//----------------------------------------------------------------------------------- \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 20334ac..ce05fff 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -292,6 +292,7 @@ KW_TRANSACTIONS: 'TRANSACTIONS'; KW_REWRITE : 'REWRITE'; KW_AUTHORIZATION: 'AUTHORIZATION'; KW_CONF: 'CONF'; +KW_VALUES: 'VALUES'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index a76cad7..38b9b17 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -331,6 +331,15 @@ TOK_RESOURCE_LIST; TOK_COMPACT; TOK_SHOW_COMPACTIONS; TOK_SHOW_TRANSACTIONS; +TOK_DELETE_FROM; +TOK_UPDATE_TABLE; +TOK_SET_COLUMNS_CLAUSE; +TOK_VALUE_ROW; +TOK_VALUES_TABLE; +TOK_VIRTUAL_TABLE; +TOK_VIRTUAL_TABREF; +TOK_ANONYMOUS; +TOK_COL_NAME; } @@ -469,6 +478,9 @@ import java.util.HashMap; xlateMap.put("KW_DEFINED", "DEFINED"); xlateMap.put("KW_SUBQUERY", "SUBQUERY"); xlateMap.put("KW_REWRITE", "REWRITE"); + xlateMap.put("KW_UPDATE", "UPDATE"); + + xlateMap.put("KW_VALUES", "VALUES"); // Operators xlateMap.put("DOT", "."); @@ -638,6 +650,8 @@ execStatement | exportStatement | importStatement | ddlStatement + | deleteStatement + | updateStatement ; loadStatement @@ -2095,11 +2109,28 @@ singleFromStatement ( b+=body )+ -> ^(TOK_QUERY fromClause body+) ; +/* +The valuesClause rule below ensures that the parse tree for +"insert into table FOO values ((1,2),(3,4))" looks the same as +"insert into table FOO select a,b from values((1,2),(3,4)) as BAR(a,b)" which itself is made to look +very similar to the tree for "insert into table FOO select a,b from BAR". Since virtual table name +is implicit, it's represented as TOK_ANONYMOUS. +*/ regularBody[boolean topLevel] : i=insertClause + ( s=selectStatement[topLevel] {$s.tree.getChild(1).replaceChildren(0, 0, $i.tree);} -> {$s.tree} + | + valuesClause + -> ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_VIRTUAL_TABLE ^(TOK_VIRTUAL_TABREF ^(TOK_ANONYMOUS)) valuesClause) + ) + ^(TOK_INSERT {$i.tree} ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))) + ) + ) | selectStatement[topLevel] ; @@ -2208,3 +2239,34 @@ limitClause : KW_LIMIT num=Number -> ^(TOK_LIMIT $num) ; + +//DELETE FROM WHERE ...; +deleteStatement +@init { pushMsg("delete statement", state); } +@after { popMsg(state); } + : + KW_DELETE KW_FROM tableName (whereClause)? -> ^(TOK_DELETE_FROM tableName whereClause?) + ; + +/*SET = (3 + col2)*/ +columnAssignmentClause + : + tableOrColumn EQUAL^ atomExpression + ; + +/*SET col1 = 5, col2 = (4 + col4), ...*/ +setColumnsClause + : + KW_SET columnAssignmentClause (COMMA columnAssignmentClause)* -> ^(TOK_SET_COLUMNS_CLAUSE columnAssignmentClause* ) + ; + +/* + UPDATE
+ SET col1 = val1, col2 = val2... WHERE ... +*/ +updateStatement +@init { pushMsg("update statement", state); } +@after { popMsg(state); } + : + KW_UPDATE tableName setColumnsClause whereClause? -> ^(TOK_UPDATE_TABLE tableName setColumnsClause whereClause?) + ; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7a71ec7..3b98932 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -972,6 +972,8 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) ASTNode frm = (ASTNode) ast.getChild(0); if (frm.getToken().getType() == HiveParser.TOK_TABREF) { processTable(qb, frm); + } else if (frm.getToken().getType() == HiveParser.TOK_VIRTUAL_TABLE) { + throw new RuntimeException("VALUES() clause is not fully supported yet..."); } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, frm); } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW || @@ -1164,6 +1166,10 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) case HiveParser.TOK_CTE: processCTE(qb, ast); break; + case HiveParser.TOK_DELETE_FROM: + throw new RuntimeException("DELETE is not (yet) implemented..."); + case HiveParser.TOK_UPDATE_TABLE: + throw new RuntimeException("UPDATE is not (yet) implemented..."); default: skipRecursion = false; break; diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java index 4a6dc61..f5bc427 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java @@ -49,6 +49,9 @@ public static HiveCommand find(String[] command) { if (command.length > 1 && "role".equalsIgnoreCase(command[1])) { // special handling for set role r1 statement return null; + } else if(command.length > 1 && "from".equalsIgnoreCase(command[1])) { + //special handling for SQL "delete from
where..." + return null; } else if (COMMANDS.contains(cmd)) { return HiveCommand.valueOf(cmd); } diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java new file mode 100644 index 0000000..f59a7ea --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestIUD { + private static HiveConf conf; + + private ParseDriver pd; + private SemanticAnalyzer sA; + + @BeforeClass + public static void initialize() { + conf = new HiveConf(SemanticAnalyzer.class); + SessionState.start(conf); + } + + @Before + public void setup() throws SemanticException { + pd = new ParseDriver(); + sA = new SemanticAnalyzer(conf); + } + + ASTNode parse(String query) throws ParseException { + ASTNode nd = pd.parse(query); + return (ASTNode) nd.getChild(0); + } + @Test + public void testDeleteNoWhere() throws ParseException { + ASTNode ast = parse("DELETE FROM src"); + Assert.assertEquals("AST doesn't match", + "(TOK_DELETE_FROM " + + "(TOK_TABNAME src))", ast.toStringTree()); + } + @Test + public void testDeleteWithWhere() throws ParseException { + ASTNode ast = parse("DELETE FROM src WHERE key IS NOT NULL AND src.value < 0"); + Assert.assertEquals("AST doesn't match", + "(TOK_DELETE_FROM " + + "(TOK_TABNAME src) " + + "(TOK_WHERE " + + "(AND " + + "(TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL key)) " + + "(< (. (TOK_TABLE_OR_COL src) value) 0))))", + ast.toStringTree()); + } + @Test + public void testUpdateNoWhereSingleSet() throws ParseException { + ASTNode ast = parse("UPDATE src set key = 3"); + Assert.assertEquals("AST doesn't match", + "(TOK_UPDATE_TABLE " + + "(TOK_TABNAME src) " + + "(TOK_SET_COLUMNS_CLAUSE " + + "(= " + + "(TOK_TABLE_OR_COL key) 3)))", + ast.toStringTree()); + } + @Test + public void testUpdateNoWhereMultiSet() throws ParseException { + ASTNode ast = parse("UPDATE src set key = 3, value = 8"); + Assert.assertEquals("AST doesn't match", + "(TOK_UPDATE_TABLE " + + "(TOK_TABNAME src) " + + "(TOK_SET_COLUMNS_CLAUSE " + + "(= " + + "(TOK_TABLE_OR_COL key) 3) " + + "(= " + + "(TOK_TABLE_OR_COL value) 8)))", + ast.toStringTree()); + } + @Test + public void testUpdateWithWhereSingleSet() throws ParseException { + ASTNode ast = parse("UPDATE src SET key = 3 WHERE value IS NULL"); + Assert.assertEquals("AST doesn't match", + "(TOK_UPDATE_TABLE " + + "(TOK_TABNAME src) " + + "(TOK_SET_COLUMNS_CLAUSE " + + "(= " + + "(TOK_TABLE_OR_COL key) 3)) " + + "(TOK_WHERE (TOK_FUNCTION TOK_ISNULL (TOK_TABLE_OR_COL value))))", + ast.toStringTree()); + } + @Test + public void testUpdateWithWhereMultiSet() throws ParseException { + ASTNode ast = parse("UPDATE src SET key = 3, value = 8 WHERE VALUE = 1230997"); + Assert.assertEquals("AST doesn't match", + "(TOK_UPDATE_TABLE " + + "(TOK_TABNAME src) " + + "(TOK_SET_COLUMNS_CLAUSE " + + "(= " + + "(TOK_TABLE_OR_COL key) 3) " + + "(= " + + "(TOK_TABLE_OR_COL value) 8)) " + + "(TOK_WHERE (= (TOK_TABLE_OR_COL VALUE) 1230997)))", + ast.toStringTree()); + } + @Test + public void testStandardInsertIntoTable() throws ParseException { + ASTNode ast = parse("INSERT into TABLE page_view SELECT pvs.viewTime, pvs.userid from page_view_stg pvs where pvs.userid is null"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_TABREF (TOK_TABNAME page_view_stg) pvs)) " + + "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " + + "(TOK_SELECT " + + "(TOK_SELEXPR (. (TOK_TABLE_OR_COL pvs) viewTime)) " + + "(TOK_SELEXPR (. (TOK_TABLE_OR_COL pvs) userid))) " + + "(TOK_WHERE (TOK_FUNCTION TOK_ISNULL (. (TOK_TABLE_OR_COL pvs) userid)))))", + ast.toStringTree()); + } + @Test + public void testSelectStarFromVirtTable1Row() throws ParseException { + ASTNode ast = parse("select * from values ((3,4)) as VC(a,b)"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF (TOK_TABNAME VC) (TOK_COL_NAME a b)) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW 3 4)))) " + + "(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))", + ast.toStringTree()); + } + @Test + public void testSelectStarFromVirtTable2Row() throws ParseException { + ASTNode ast = parse("select * from values ((1,2),(3,4)) as VC(a,b)"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF (TOK_TABNAME VC) (TOK_COL_NAME a b)) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW 1 2) (TOK_VALUE_ROW 3 4)))) " + + "(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))", + ast.toStringTree()); + } + @Test + public void testSelectStarFromVirtTable2RowNamedProjections() throws ParseException { + ASTNode ast = parse("select a as c, b as d from values ((1,2),(3,4)) as VC(a,b)"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF (TOK_TABNAME VC) (TOK_COL_NAME a b)) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW 1 2) (TOK_VALUE_ROW 3 4)))) " + + "(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) " + + "(TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL a) c) (TOK_SELEXPR (TOK_TABLE_OR_COL b) d))))", + ast.toStringTree()); + } + @Test + public void testInsertIntoTableAsSelectFromNamedVirtTable() throws ParseException { + ASTNode ast = parse("insert into table page_view select a,b as c from values ((1,2),(3,4)) as VC(a,b) where b = 9"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF (TOK_TABNAME VC) (TOK_COL_NAME a b)) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW 1 2) (TOK_VALUE_ROW 3 4)))) " + + "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " + + "(TOK_SELECT " + + "(TOK_SELEXPR (TOK_TABLE_OR_COL a)) " + + "(TOK_SELEXPR (TOK_TABLE_OR_COL b) c)) " + + "(TOK_WHERE (= (TOK_TABLE_OR_COL b) 9))))", + ast.toStringTree()); + } + @Test + public void testInsertIntoTableFromAnonymousTable() throws ParseException { + ASTNode ast = parse("insert into table page_view values((1,2),(3,4))"); + Assert.assertEquals("AST doesn't match", + "(TOK_QUERY " + + "(TOK_FROM " + + "(TOK_VIRTUAL_TABLE " + + "(TOK_VIRTUAL_TABREF TOK_ANONYMOUS) " + + "(TOK_VALUES_TABLE (TOK_VALUE_ROW 1 2) (TOK_VALUE_ROW 3 4)))) " + + "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " + + "(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))", + ast.toStringTree()); + } +}