Index: ql/src/test/results/clientnegative/invalid_tbl_name.q.out =================================================================== --- ql/src/test/results/clientnegative/invalid_tbl_name.q.out (revision 723411) +++ ql/src/test/results/clientnegative/invalid_tbl_name.q.out (working copy) @@ -1,2 +1,2 @@ -FAILED: Parse Error: line 1:20 mismatched input '-' expecting EOF +FAILED: Parse Error: line 1:20 mismatched input '-' expecting EOF in ddl statement Index: ql/src/test/results/clientnegative/invalid_create_tbl2.q.out =================================================================== --- ql/src/test/results/clientnegative/invalid_create_tbl2.q.out (revision 0) +++ ql/src/test/results/clientnegative/invalid_create_tbl2.q.out (revision 0) @@ -0,0 +1,2 @@ +FAILED: Parse Error: line 1:7 mismatched input 'tabl' expecting KW_TEMPORARY in create statement + Index: ql/src/test/results/clientnegative/invalid_select_expression.q.out =================================================================== --- ql/src/test/results/clientnegative/invalid_select_expression.q.out (revision 0) +++ ql/src/test/results/clientnegative/invalid_select_expression.q.out (revision 0) @@ -0,0 +1,2 @@ +FAILED: Parse Error: line 1:32 cannot recognize input '.' in table column identifier + Index: ql/src/test/queries/clientnegative/invalid_create_tbl2.q =================================================================== --- ql/src/test/queries/clientnegative/invalid_create_tbl2.q (revision 0) +++ ql/src/test/queries/clientnegative/invalid_create_tbl2.q (revision 0) @@ -0,0 +1,3 @@ +create tabl tmp_zshao_22 (id int, name strin; + + Index: ql/src/test/queries/clientnegative/invalid_select_expression.q =================================================================== --- ql/src/test/queries/clientnegative/invalid_select_expression.q (revision 0) +++ ql/src/test/queries/clientnegative/invalid_select_expression.q (revision 0) @@ -0,0 +1 @@ +select foo from a a where foo > .foo; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 722582) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -5,6 +5,7 @@ output=AST; ASTLabelType=CommonTree; backtrack=true; +k=1; } tokens { @@ -125,9 +126,14 @@ } @lexer::header {package org.apache.hadoop.hive.ql.parse;} + +@members { + Stack msgs = new Stack(); +} + @rulecatch { catch (RecognitionException e) { - reportError(e); + reportError(e); throw e; } } @@ -139,6 +145,8 @@ ; explainStatement +@init { msgs.push("explain statement"); } +@after { msgs.pop(); } : KW_EXPLAIN (isExtended=KW_EXTENDED)? execStatement -> ^(TOK_EXPLAIN execStatement $isExtended?) ; @@ -149,11 +157,15 @@ ; loadStatement +@init { msgs.push("load statement"); } +@after { msgs.pop(); } : KW_LOAD KW_DATA (islocal=KW_LOCAL)? KW_INPATH (path=StringLiteral) (isoverwrite=KW_OVERWRITE)? KW_INTO KW_TABLE (tab=tabName) -> ^(TOK_LOAD $path $tab $islocal? $isoverwrite?) ; ddlStatement +@init { msgs.push("ddl statement"); } +@after { msgs.pop(); } : createStatement | dropStatement | alterStatement @@ -162,17 +174,28 @@ | createFunctionStatement ; +createFunctionStatement + : KW_CREATE KW_TEMPORARY KW_FUNCTION Identifier KW_AS StringLiteral + -> ^(TOK_CREATEFUNCTION Identifier StringLiteral) + ; + createStatement +@init { msgs.push("create statement"); } +@after { msgs.pop(); } : KW_CREATE (ext=KW_EXTERNAL)? KW_TABLE name=Identifier (LPAREN columnNameTypeList RPAREN)? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation? -> {$ext == null}? ^(TOK_CREATETABLE $name columnNameTypeList? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) -> ^(TOK_CREATEEXTTABLE $name columnNameTypeList? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) ; dropStatement +@init { msgs.push("drop statement"); } +@after { msgs.pop(); } : KW_DROP KW_TABLE Identifier -> ^(TOK_DROPTABLE Identifier) ; alterStatement +@init { msgs.push("alter statement"); } +@after { msgs.pop(); } : alterStatementRename | alterStatementAddCol | alterStatementDropPartitions @@ -181,27 +204,37 @@ ; alterStatementRename +@init { msgs.push("rename statement"); } +@after { msgs.pop(); } : KW_ALTER KW_TABLE oldName=Identifier KW_RENAME KW_TO newName=Identifier -> ^(TOK_ALTERTABLE_RENAME $oldName $newName) ; alterStatementAddCol +@init { msgs.push("add column statement"); } +@after { msgs.pop(); } : KW_ALTER KW_TABLE Identifier (add=KW_ADD | replace=KW_REPLACE) KW_COLUMNS LPAREN columnNameTypeList RPAREN -> {$add != null}? ^(TOK_ALTERTABLE_ADDCOLS Identifier columnNameTypeList) -> ^(TOK_ALTERTABLE_REPLACECOLS Identifier columnNameTypeList) ; alterStatementDropPartitions +@init { msgs.push("drop partition statement"); } +@after { msgs.pop(); } : KW_ALTER KW_TABLE Identifier KW_DROP partitionSpec (COMMA partitionSpec)* -> ^(TOK_ALTERTABLE_DROPPARTS Identifier partitionSpec+) ; alterStatementProperties +@init { msgs.push("alter properties statement"); } +@after { msgs.pop(); } : KW_ALTER KW_TABLE name=Identifier KW_SET KW_PROPERTIES tableProperties -> ^(TOK_ALTERTABLE_PROPERTIES $name tableProperties) ; alterStatementSerdeProperties +@init { msgs.push("alter serdes statement"); } +@after { msgs.pop(); } : KW_ALTER KW_TABLE name=Identifier KW_SET KW_SERDE serde=StringLiteral (KW_WITH KW_SERDEPROPERTIES tableProperties)? -> ^(TOK_ALTERTABLE_SERIALIZER $name $serde tableProperties?) | KW_ALTER KW_TABLE name=Identifier KW_SET KW_SERDEPROPERTIES tableProperties @@ -209,49 +242,63 @@ ; tabTypeExpr +@init { msgs.push("specifying table types"); } +@after { msgs.pop(); } : Identifier (DOT^ (Identifier | KW_ELEM_TYPE | KW_KEY_TYPE | KW_VALUE_TYPE))* ; partTypeExpr +@init { msgs.push("specifying table partitions"); } +@after { msgs.pop(); } : tabTypeExpr partitionSpec? -> ^(TOK_TABTYPE tabTypeExpr partitionSpec?) ; descStatement +@init { msgs.push("describe statement"); } +@after { msgs.pop(); } : KW_DESCRIBE (isExtended=KW_EXTENDED)? (parttype=partTypeExpr) -> ^(TOK_DESCTABLE $parttype $isExtended?) ; showStatement +@init { msgs.push("show statement"); } +@after { msgs.pop(); } : KW_SHOW KW_TABLES showStmtIdentifier? -> ^(TOK_SHOWTABLES showStmtIdentifier?) | KW_SHOW KW_PARTITIONS Identifier -> ^(TOK_SHOWPARTITIONS Identifier) ; -createFunctionStatement - : KW_CREATE KW_TEMPORARY KW_FUNCTION Identifier KW_AS StringLiteral - -> ^(TOK_CREATEFUNCTION Identifier StringLiteral) - ; showStmtIdentifier +@init { msgs.push("identifier for show statement"); } +@after { msgs.pop(); } : Identifier | StringLiteral ; tableComment +@init { msgs.push("table's comment"); } +@after { msgs.pop(); } : KW_COMMENT comment=StringLiteral -> ^(TOK_TABLECOMMENT $comment) ; tablePartition +@init { msgs.push("table partition specification"); } +@after { msgs.pop(); } : KW_PARTITIONED KW_BY LPAREN columnNameTypeList RPAREN -> ^(TOK_TABLEPARTCOLS columnNameTypeList) ; tableBuckets +@init { msgs.push("table buckets specification"); } +@after { msgs.pop(); } : KW_CLUSTERED KW_BY LPAREN bucketCols=columnNameList RPAREN (KW_SORTED KW_BY LPAREN sortCols=columnNameOrderList RPAREN)? KW_INTO num=Number KW_BUCKETS -> ^(TOK_TABLEBUCKETS $bucketCols $sortCols? $num) ; tableRowFormat +@init { msgs.push("table row format specification"); } +@after { msgs.pop(); } : KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier? -> ^(TOK_TABLEROWFORMAT tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?) @@ -260,91 +307,125 @@ ; tableProperties +@init { msgs.push("table properties"); } +@after { msgs.pop(); } : LPAREN propertiesList RPAREN -> ^(TOK_TABLEPROPERTIES propertiesList) ; propertiesList +@init { msgs.push("properties list"); } +@after { msgs.pop(); } : keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_TABLEPROPLIST keyValueProperty+) ; keyValueProperty +@init { msgs.push("specifying key/value property"); } +@after { msgs.pop(); } : key=StringLiteral EQUAL value=StringLiteral -> ^(TOK_TABLEPROPERTY $key $value) ; tableRowFormatFieldIdentifier +@init { msgs.push("table row format's field separator"); } +@after { msgs.pop(); } : KW_FIELDS KW_TERMINATED KW_BY fldIdnt=StringLiteral -> ^(TOK_TABLEROWFORMATFIELD $fldIdnt) ; tableRowFormatCollItemsIdentifier +@init { msgs.push("table row format's column separator"); } +@after { msgs.pop(); } : KW_COLLECTION KW_ITEMS KW_TERMINATED KW_BY collIdnt=StringLiteral -> ^(TOK_TABLEROWFORMATCOLLITEMS $collIdnt) ; tableRowFormatMapKeysIdentifier +@init { msgs.push("table row format's map key separator"); } +@after { msgs.pop(); } : KW_MAP KW_KEYS KW_TERMINATED KW_BY mapKeysIdnt=StringLiteral -> ^(TOK_TABLEROWFORMATMAPKEYS $mapKeysIdnt) ; tableRowFormatLinesIdentifier +@init { msgs.push("table row format's line separator"); } +@after { msgs.pop(); } : KW_LINES KW_TERMINATED KW_BY linesIdnt=StringLiteral -> ^(TOK_TABLEROWFORMATLINES $linesIdnt) ; tableFileFormat +@init { msgs.push("table file format specification"); } +@after { msgs.pop(); } : KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE ; tableLocation +@init { msgs.push("table location specification"); } +@after { msgs.pop(); } : KW_LOCATION locn=StringLiteral -> ^(TOK_TABLELOCATION $locn) ; columnNameTypeList +@init { msgs.push("column name type list"); } +@after { msgs.pop(); } : columnNameType (COMMA columnNameType)* -> ^(TOK_TABCOLLIST columnNameType+) ; columnNameList +@init { msgs.push("column name list"); } +@after { msgs.pop(); } : columnName (COMMA columnName)* -> ^(TOK_TABCOLNAME columnName+) ; columnName +@init { msgs.push("column name"); } +@after { msgs.pop(); } : Identifier ; columnNameOrderList +@init { msgs.push("column name order list"); } +@after { msgs.pop(); } : columnNameOrder (COMMA columnNameOrder)* -> ^(TOK_TABCOLNAME columnNameOrder+) ; columnNameOrder +@init { msgs.push("column name order"); } +@after { msgs.pop(); } : Identifier (asc=KW_ASC | desc=KW_DESC)? -> {$desc == null}? ^(TOK_TABSORTCOLNAMEASC Identifier) -> ^(TOK_TABSORTCOLNAMEDESC Identifier) ; columnNameType +@init { msgs.push("column name type"); } +@after { msgs.pop(); } : colName=Identifier colType (KW_COMMENT comment=StringLiteral)? -> {$comment == null}? ^(TOK_TABCOL $colName colType) -> ^(TOK_TABCOL $colName colType $comment) ; colType +@init { msgs.push("column type"); } +@after { msgs.pop(); } : primitiveType | listType | mapType ; primitiveType +@init { msgs.push("primitive type specification"); } +@after { msgs.pop(); } : KW_TINYINT -> TOK_TINYINT | KW_SMALLINT -> TOK_SMALLINT | KW_INT -> TOK_INT @@ -359,15 +440,21 @@ ; listType +@init { msgs.push("list type"); } +@after { msgs.pop(); } : KW_ARRAY LESSTHAN primitiveType GREATERTHAN -> ^(TOK_LIST primitiveType) ; mapType +@init { msgs.push("map type"); } +@after { msgs.pop(); } : KW_MAP LESSTHAN left=primitiveType COMMA right=primitiveType GREATERTHAN -> ^(TOK_MAP $left $right) ; queryOperator +@init { msgs.push("query operator"); } +@after { msgs.pop(); } : KW_UNION KW_ALL -> ^(TOK_UNION) ; @@ -423,11 +510,15 @@ ; insertClause +@init { msgs.push("insert clause"); } +@after { msgs.pop(); } : KW_INSERT KW_OVERWRITE destination -> ^(TOK_DESTINATION destination) ; destination +@init { msgs.push("destination specification"); } +@after { msgs.pop(); } : KW_LOCAL KW_DIRECTORY StringLiteral -> ^(TOK_LOCAL_DIR StringLiteral) | KW_DIRECTORY StringLiteral -> ^(TOK_DIR StringLiteral) @@ -435,6 +526,8 @@ ; limitClause +@init { msgs.push("limit clause"); } +@after { msgs.pop(); } : KW_LIMIT num=Number -> ^(TOK_LIMIT $num) ; @@ -442,6 +535,8 @@ //----------------------- Rules for parsing selectClause ----------------------------- // select a,b,c ... selectClause +@init { msgs.push("select clause"); } +@after { msgs.pop(); } : KW_SELECT (KW_ALL | dist=KW_DISTINCT)? selectList -> {$dist == null}? ^(TOK_SELECT selectList) @@ -449,17 +544,23 @@ ; selectList +@init { msgs.push("select list"); } +@after { msgs.pop(); } : selectItem ( COMMA selectItem )* -> selectItem+ | trfmClause -> ^(TOK_SELEXPR trfmClause) ; selectItem +@init { msgs.push("selection target"); } +@after { msgs.pop(); } : ( selectExpression (KW_AS Identifier)?) -> ^(TOK_SELEXPR selectExpression Identifier?) ; trfmClause +@init { msgs.push("from clause"); } +@after { msgs.pop(); } : KW_TRANSFORM LPAREN expressionList RPAREN @@ -469,6 +570,8 @@ ; selectExpression +@init { msgs.push("select expression"); } +@after { msgs.pop(); } : expression | tableAllColumns ; @@ -483,16 +586,22 @@ // table.column tableColumn +@init { msgs.push("table column identifier"); } +@after { msgs.pop(); } : (tab=Identifier DOT)? col=Identifier -> ^(TOK_COLREF $tab? $col) ; expressionList +@init { msgs.push("expression list"); } +@after { msgs.pop(); } : expression (COMMA expression)* -> ^(TOK_EXPLIST expression+) ; aliasList +@init { msgs.push("alias list"); } +@after { msgs.pop(); } : Identifier (COMMA Identifier)* -> ^(TOK_ALIASLIST Identifier+) ; @@ -500,18 +609,24 @@ //----------------------- Rules for parsing fromClause ------------------------------ // from [col1, col2, col3] table1, [col4, col5] table2 fromClause +@init { msgs.push("from clause"); } +@after { msgs.pop(); } : KW_FROM joinSource -> ^(TOK_FROM joinSource) | KW_FROM fromSource -> ^(TOK_FROM fromSource) ; joinSource +@init { msgs.push("join source"); } +@after { msgs.pop(); } : fromSource ( joinToken^ fromSource (KW_ON! expression)? )+ ; joinToken +@init { msgs.push("join type specifier"); } +@after { msgs.pop(); } : KW_JOIN -> TOK_JOIN | KW_LEFT KW_OUTER KW_JOIN -> TOK_LEFTOUTERJOIN @@ -520,22 +635,30 @@ ; fromSource +@init { msgs.push("from source"); } +@after { msgs.pop(); } : (tableSource | subQuerySource) ; tableSample +@init { msgs.push("table sample specification"); } +@after { msgs.pop(); } : KW_TABLESAMPLE LPAREN KW_BUCKET (numerator=Number) KW_OUT KW_OF (denominator=Number) (KW_ON expr+=expression (COMMA expr+=expression)*)? RPAREN -> ^(TOK_TABLESAMPLE $numerator $denominator $expr*) ; tableSource +@init { msgs.push("table source"); } +@after { msgs.pop(); } : tabname=Identifier (ts=tableSample)? (alias=Identifier)? -> ^(TOK_TABREF $tabname $ts? $alias?) ; subQuerySource +@init { msgs.push("subquery source"); } +@after { msgs.pop(); } : LPAREN queryStatementExpression RPAREN Identifier -> ^(TOK_SUBQUERY queryStatementExpression Identifier) ; @@ -543,11 +666,15 @@ //----------------------- Rules for parsing whereClause ----------------------------- // where a=b and ... whereClause +@init { msgs.push("where clause"); } +@after { msgs.pop(); } : KW_WHERE searchCondition -> ^(TOK_WHERE searchCondition) ; searchCondition +@init { msgs.push("search condition"); } +@after { msgs.pop(); } : expression ; @@ -556,6 +683,8 @@ // group by a,b groupByClause +@init { msgs.push("group by clause"); } +@after { msgs.pop(); } : KW_GROUP KW_BY groupByExpression @@ -564,12 +693,16 @@ ; groupByExpression +@init { msgs.push("group by expression"); } +@after { msgs.pop(); } : expression ; // order by a,b orderByClause +@init { msgs.push("order by clause"); } +@after { msgs.pop(); } : KW_ORDER KW_BY orderByExpression @@ -578,12 +711,16 @@ ; orderByExpression +@init { msgs.push("order by expression"); } +@after { msgs.pop(); } : expression (KW_ASC | KW_DESC)? ; clusterByClause +@init { msgs.push("cluster by clause"); } +@after { msgs.pop(); } : KW_CLUSTER KW_BY Identifier @@ -591,12 +728,16 @@ ; clusterByExpression +@init { msgs.push("cluster by expression"); } +@after { msgs.pop(); } : expression ; // fun(par1, par2, par3) function +@init { msgs.push("function specification"); } +@after { msgs.pop(); } : // LEFT and RIGHT keywords are also function names Identifier LPAREN ( @@ -610,6 +751,8 @@ ; castExpression +@init { msgs.push("cast expression"); } +@after { msgs.pop(); } : KW_CAST LPAREN @@ -620,6 +763,8 @@ ; constant +@init { msgs.push("constant"); } +@after { msgs.pop(); } : Number | StringLiteral @@ -628,15 +773,21 @@ ; charSetStringLiteral +@init { msgs.push("character string literal"); } +@after { msgs.pop(); } : csName=CharSetName csLiteral=CharSetLiteral -> ^(TOK_CHARSETLITERAL $csName $csLiteral) ; -expression: +expression +@init { msgs.push("expression specification"); } +@after { msgs.pop(); } + : precedenceOrExpression ; -atomExpression: +atomExpression + : KW_NULL -> TOK_NULL | constant | function Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (revision 722582) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (working copy) @@ -236,6 +236,9 @@ msg = super.getErrorMessage(e, xlateNames); } + if(msgs.size() > 0) { + msg = msg + " in " + msgs.peek(); + } return msg; }