diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7e5a515..36e5a5f 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1954,7 +1954,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { " none: default(past) behavior. Implies only alphaNumeric and underscore are valid characters in identifiers.\n" + " column: implies column names can contain any character." ), - + HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS("hive.support.sql11.reserved.keywords", false, + "This flag should be set to true to enable support for SQL2011 reserved keywords.\n" + + "The default value is true."), // role names are case-insensitive USERS_IN_ADMIN_ROLE("hive.users.in.admin.role", "", false, "Comma separated list of users who are in admin role for bootstrapping.\n" + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index b72ee5d..038ed99 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -35,6 +35,9 @@ k=3; RecognitionException e) { gParent.errors.add(new ParseError(gParent, e, tokenNames)); } + protected boolean useSQL11ReservedKeywordsForIdentifier() { + return gParent.useSQL11ReservedKeywordsForIdentifier(); + } } @rulecatch { @@ -126,7 +129,7 @@ lateralView @init {gParent.pushMsg("lateral view", state); } @after {gParent.popMsg(state); } : - KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)? + (KW_LATERAL KW_VIEW KW_OUTER) => KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)? -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias))) | KW_LATERAL KW_VIEW function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)? @@ -177,7 +180,12 @@ tableSample tableSource @init { gParent.pushMsg("table source", state); } @after { gParent.popMsg(state); } - : tabname=tableName (props=tableProperties)? (ts=tableSample)? (KW_AS? alias=Identifier)? + : tabname=tableName + ((tableProperties) => props=tableProperties)? + ((tableSample) => ts=tableSample)? + ((KW_AS) => (KW_AS alias=Identifier) + | + (Identifier) => (alias=Identifier))? -> ^(TOK_TABREF $tabname $props? $ts? $alias?) ; @@ -232,11 +240,11 @@ partitionedTableFunction @init { gParent.pushMsg("ptf clause", state); } @after { gParent.popMsg(state); } : - name=Identifier - LPAREN KW_ON ptfsrc=partitionTableFunctionSource partitioningSpec? - ((Identifier LPAREN expression RPAREN ) => Identifier LPAREN expression RPAREN ( COMMA Identifier LPAREN expression RPAREN)*)? - RPAREN alias=Identifier? - -> ^(TOK_PTBLFUNCTION $name $alias? partitionTableFunctionSource partitioningSpec? expression*) + name=Identifier LPAREN KW_ON + ((partitionTableFunctionSource) => (ptfsrc=partitionTableFunctionSource spec=partitioningSpec?)) + ((Identifier LPAREN expression RPAREN ) => Identifier LPAREN expression RPAREN ( COMMA Identifier LPAREN expression RPAREN)*)? + ((RPAREN) => (RPAREN)) ((Identifier) => alias=Identifier)? + -> ^(TOK_PTBLFUNCTION $name $alias? $ptfsrc $spec? expression*) ; //----------------------- Rules for parsing whereClause ----------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 90b84ac..e7de6c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -42,7 +42,6 @@ KW_TRUE : 'TRUE'; KW_FALSE : 'FALSE'; KW_ALL : 'ALL'; KW_NONE: 'NONE'; -KW_DEFAULT : 'DEFAULT'; KW_AND : 'AND'; KW_OR : 'OR'; KW_NOT : 'NOT' | '!'; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 2105e19..d2d9989 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -354,6 +354,8 @@ package org.apache.hadoop.hive.ql.parse; import java.util.Collection; import java.util.HashMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; } @@ -370,7 +372,6 @@ import java.util.HashMap; xlateMap.put("KW_FALSE", "FALSE"); xlateMap.put("KW_ALL", "ALL"); xlateMap.put("KW_NONE", "NONE"); - xlateMap.put("KW_DEFAULT", "DEFAULT"); xlateMap.put("KW_AND", "AND"); xlateMap.put("KW_OR", "OR"); xlateMap.put("KW_NOT", "NOT"); @@ -620,6 +621,13 @@ import java.util.HashMap; private CommonTree throwSetOpException() throws RecognitionException { throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.", ""); } + private Configuration hiveConf; + public void setHiveConf(Configuration hiveConf) { + this.hiveConf = hiveConf; + } + protected boolean useSQL11ReservedKeywordsForIdentifier() { + return !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS); + } } @rulecatch { @@ -712,8 +720,8 @@ ddlStatement | unlockDatabase | createRoleStatement | dropRoleStatement - | grantPrivileges - | revokePrivileges + | (grantPrivileges) => grantPrivileges + | (revokePrivileges) => revokePrivileges | showGrants | showRoleGrants | showRolePrincipals @@ -955,8 +963,7 @@ alterStatement alterTableStatementSuffix @init { pushMsg("alter table statement", state); } @after { popMsg(state); } - : alterStatementSuffixRename[true] - | alterStatementSuffixUpdateStatsCol + : (alterStatementSuffixRename[true]) => alterStatementSuffixRename[true] | alterStatementSuffixDropPartitions[true] | alterStatementSuffixAddPartitions[true] | alterStatementSuffixTouch @@ -1297,15 +1304,15 @@ fileFormat tabTypeExpr @init { pushMsg("specifying table types", state); } @after { popMsg(state); } - - : identifier (DOT^ (KW_ELEM_TYPE | KW_KEY_TYPE | KW_VALUE_TYPE | identifier))* - ; - -descTabTypeExpr -@init { pushMsg("specifying describe table types", state); } -@after { popMsg(state); } - - : identifier (DOT^ (KW_ELEM_TYPE | KW_KEY_TYPE | KW_VALUE_TYPE | identifier))* identifier? + : identifier (DOT^ + ( + (KW_ELEM_TYPE) => KW_ELEM_TYPE + | + (KW_KEY_TYPE) => KW_KEY_TYPE + | + (KW_VALUE_TYPE) => KW_VALUE_TYPE + | identifier + ))* identifier? ; partTypeExpr @@ -1314,21 +1321,22 @@ partTypeExpr : tabTypeExpr partitionSpec? -> ^(TOK_TABTYPE tabTypeExpr partitionSpec?) ; -descPartTypeExpr -@init { pushMsg("specifying describe table partitions", state); } -@after { popMsg(state); } - : descTabTypeExpr partitionSpec? -> ^(TOK_TABTYPE descTabTypeExpr partitionSpec?) - ; - descStatement @init { pushMsg("describe statement", state); } @after { popMsg(state); } - : (KW_DESCRIBE|KW_DESC) (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) - | (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?) - | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) + : + (KW_DESCRIBE|KW_DESC) + ( + (KW_DATABASE|KW_SCHEMA) => (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) + | + (KW_FUNCTION) => KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) + | + (KW_FORMATTED|KW_EXTENDED|KW_PRETTY) => ((descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY) parttype=partTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions) + | + parttype=partTypeExpr -> ^(TOK_DESCTABLE $parttype) + ) ; - analyzeStatement @init { pushMsg("analyze statement", state); } @after { popMsg(state); } @@ -1350,8 +1358,12 @@ showStatement | KW_SHOW KW_TABLE KW_EXTENDED ((KW_FROM|KW_IN) db_name=identifier)? KW_LIKE showStmtIdentifier partitionSpec? -> ^(TOK_SHOW_TABLESTATUS showStmtIdentifier $db_name? partitionSpec?) | KW_SHOW KW_TBLPROPERTIES tableName (LPAREN prptyName=StringLiteral RPAREN)? -> ^(TOK_SHOW_TBLPROPERTIES tableName $prptyName?) - | KW_SHOW KW_LOCKS (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?) - | KW_SHOW KW_LOCKS (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) + | KW_SHOW KW_LOCKS + ( + (KW_DATABASE|KW_SCHEMA) => (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) + | + (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?) + ) | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) | KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS) @@ -1459,8 +1471,12 @@ showCurrentRole setRole @init {pushMsg("set role", state);} @after {popMsg(state);} - : KW_SET KW_ROLE roleName=identifier - -> ^(TOK_SHOW_SET_ROLE $roleName) + : KW_SET KW_ROLE + ( + (KW_ALL) => (all=KW_ALL) -> ^(TOK_SHOW_SET_ROLE Identifier[$all.text]) + | + identifier -> ^(TOK_SHOW_SET_ROLE identifier) + ) ; showGrants @@ -1481,7 +1497,7 @@ showRolePrincipals privilegeIncludeColObject @init {pushMsg("privilege object including columns", state);} @after {popMsg(state);} - : KW_ALL -> ^(TOK_RESOURCE_ALL) + : (KW_ALL) => KW_ALL -> ^(TOK_RESOURCE_ALL) | privObjectCols -> ^(TOK_PRIV_OBJECT_COL privObjectCols) ; @@ -1720,7 +1736,7 @@ tableSkewed @init { pushMsg("table skewed specification", state); } @after { popMsg(state); } : - KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN (storedAsDirs)? + KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN ((storedAsDirs) => storedAsDirs)? -> ^(TOK_TABLESKEWED $skewedCols $skewedValues storedAsDirs?) ; @@ -1851,7 +1867,7 @@ tableFileFormat @init { pushMsg("table file format specification", state); } @after { popMsg(state); } : - KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + (KW_STORED KW_AS KW_INPUTFORMAT) => KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) | KW_STORED KW_BY storageHandler=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? @@ -2231,7 +2247,7 @@ simpleSelectStatement whereClause? groupByClause? havingClause? - window_clause? + ((window_clause) => window_clause)? -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? havingClause? window_clause?)) ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index cabf971..0a05ceb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -35,6 +35,9 @@ k=3; RecognitionException e) { gParent.errors.add(new ParseError(gParent, e, tokenNames)); } + protected boolean useSQL11ReservedKeywordsForIdentifier() { + return gParent.useSQL11ReservedKeywordsForIdentifier(); + } } @rulecatch { @@ -51,40 +54,41 @@ groupByClause @after { gParent.popMsg(state); } : KW_GROUP KW_BY - groupByExpression - ( COMMA groupByExpression )* + expression + ( COMMA expression)* ((rollup=KW_WITH KW_ROLLUP) | (cube=KW_WITH KW_CUBE)) ? (sets=KW_GROUPING KW_SETS LPAREN groupingSetExpression ( COMMA groupingSetExpression)* RPAREN ) ? - -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY groupByExpression+) - -> {cube != null}? ^(TOK_CUBE_GROUPBY groupByExpression+) - -> {sets != null}? ^(TOK_GROUPING_SETS groupByExpression+ groupingSetExpression+) - -> ^(TOK_GROUPBY groupByExpression+) + -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expression+) + -> {cube != null}? ^(TOK_CUBE_GROUPBY expression+) + -> {sets != null}? ^(TOK_GROUPING_SETS expression+ groupingSetExpression+) + -> ^(TOK_GROUPBY expression+) ; groupingSetExpression @init {gParent.pushMsg("grouping set expression", state); } @after {gParent.popMsg(state); } : - groupByExpression - -> ^(TOK_GROUPING_SETS_EXPRESSION groupByExpression) + (LPAREN) => groupingSetExpressionMultiple | + groupingExpressionSingle + ; + +groupingSetExpressionMultiple +@init {gParent.pushMsg("grouping set part expression", state); } +@after {gParent.popMsg(state); } + : LPAREN - groupByExpression (COMMA groupByExpression)* + expression? (COMMA expression)* RPAREN - -> ^(TOK_GROUPING_SETS_EXPRESSION groupByExpression+) - | - LPAREN - RPAREN - -> ^(TOK_GROUPING_SETS_EXPRESSION) + -> ^(TOK_GROUPING_SETS_EXPRESSION expression*) ; - -groupByExpression -@init { gParent.pushMsg("group by expression", state); } +groupingExpressionSingle +@init { gParent.pushMsg("groupingExpression expression", state); } @after { gParent.popMsg(state); } : - expression + expression -> ^(TOK_GROUPING_SETS_EXPRESSION expression) ; havingClause @@ -101,6 +105,26 @@ havingCondition expression ; +expressionsInParenthese + : + LPAREN expression (COMMA expression)* RPAREN -> expression+ + ; + +expressionsNotInParenthese + : + expression (COMMA expression)* -> expression+ + ; + +columnRefOrderInParenthese + : + LPAREN columnRefOrder (COMMA columnRefOrder)* RPAREN -> columnRefOrder+ + ; + +columnRefOrderNotInParenthese + : + columnRefOrder (COMMA columnRefOrder)* -> columnRefOrder+ + ; + // order by a,b orderByClause @init { gParent.pushMsg("order by clause", state); } @@ -108,17 +132,17 @@ orderByClause : KW_ORDER KW_BY columnRefOrder ( COMMA columnRefOrder)* -> ^(TOK_ORDERBY columnRefOrder+) ; - + clusterByClause @init { gParent.pushMsg("cluster by clause", state); } @after { gParent.popMsg(state); } : KW_CLUSTER KW_BY - LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_CLUSTERBY expression+) + ( + (LPAREN) => expressionsInParenthese -> ^(TOK_CLUSTERBY expressionsInParenthese) | - KW_CLUSTER KW_BY - expression - ( (COMMA)=>COMMA expression )* -> ^(TOK_CLUSTERBY expression+) + expressionsNotInParenthese -> ^(TOK_CLUSTERBY expressionsNotInParenthese) + ) ; partitionByClause @@ -126,10 +150,11 @@ partitionByClause @after { gParent.popMsg(state); } : KW_PARTITION KW_BY - LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_DISTRIBUTEBY expression+) + ( + (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese) | - KW_PARTITION KW_BY - expression ((COMMA)=> COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+) + expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese) + ) ; distributeByClause @@ -137,10 +162,11 @@ distributeByClause @after { gParent.popMsg(state); } : KW_DISTRIBUTE KW_BY - LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_DISTRIBUTEBY expression+) + ( + (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese) | - KW_DISTRIBUTE KW_BY - expression ((COMMA)=> COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+) + expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese) + ) ; sortByClause @@ -148,12 +174,11 @@ sortByClause @after { gParent.popMsg(state); } : KW_SORT KW_BY - LPAREN columnRefOrder - ( COMMA columnRefOrder)* RPAREN -> ^(TOK_SORTBY columnRefOrder+) + ( + (LPAREN) => columnRefOrderInParenthese -> ^(TOK_SORTBY columnRefOrderInParenthese) | - KW_SORT KW_BY - columnRefOrder - ( (COMMA)=> COMMA columnRefOrder)* -> ^(TOK_SORTBY columnRefOrder+) + columnRefOrderNotInParenthese -> ^(TOK_SORTBY columnRefOrderNotInParenthese) + ) ; // fun(par1, par2, par3) @@ -164,7 +189,7 @@ function functionName LPAREN ( - (star=STAR) + (STAR) => (star=STAR) | (dist=KW_DISTINCT)? (selectExpression (COMMA selectExpression)*)? ) RPAREN (KW_OVER ws=window_specification)? @@ -173,29 +198,15 @@ function -> ^(TOK_FUNCTIONDI functionName (selectExpression+)?) ; -nonParenthesizedFunction -@init { gParent.pushMsg("non-parenthesized function name", state); } -@after { gParent.popMsg(state); } - : - nonParenthesizedFunctionName - -> ^(TOK_FUNCTION nonParenthesizedFunctionName) - ; - -nonParenthesizedFunctionName -@init { gParent.pushMsg("non-parenthesized function name", state); } -@after { gParent.popMsg(state); } - : - KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP - ; - functionName @init { gParent.pushMsg("function name", state); } @after { gParent.popMsg(state); } : // Keyword IF is also a function name - KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | functionIdentifier + (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE) => (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE) + | + (functionIdentifier) => functionIdentifier | - // This allows current_timestamp() to work as well as current_timestamp - nonParenthesizedFunctionName + {!useSQL11ReservedKeywordsForIdentifier()}? sql11ReservedKeywordsUsedAsCastFunctionName -> Identifier[$sql11ReservedKeywordsUsedAsCastFunctionName.text] ; castExpression @@ -267,6 +278,8 @@ dateLiteral // This makes the dateLiteral more consistent with the other type literals. adaptor.create(TOK_DATELITERAL, $StringLiteral.text) } + | + KW_CURRENT_DATE -> ^(TOK_FUNCTION KW_CURRENT_DATE) ; timestampLiteral @@ -275,6 +288,8 @@ timestampLiteral { adaptor.create(TOK_TIMESTAMPLITERAL, $StringLiteral.text) } + | + KW_CURRENT_TIMESTAMP -> ^(TOK_FUNCTION KW_CURRENT_TIMESTAMP) ; expression @@ -286,12 +301,11 @@ expression atomExpression : - KW_NULL -> TOK_NULL - | constant + (KW_NULL) => KW_NULL -> TOK_NULL + | (constant) => constant | castExpression | caseExpression | whenExpression - | nonParenthesizedFunction | (functionName LPAREN) => function | tableOrColumn | LPAREN! expression RPAREN! @@ -543,7 +557,7 @@ sysFuncNames descFuncNames : - sysFuncNames + (sysFuncNames) => sysFuncNames | StringLiteral | functionIdentifier ; @@ -552,6 +566,9 @@ identifier : Identifier | nonReserved -> Identifier[$nonReserved.text] + // If it decides to support SQL11 reserved keywords, i.e., useSQL11ReservedKeywordsForIdentifier()=false, + // the sql11keywords in existing q tests will NOT be added back. + | {useSQL11ReservedKeywordsForIdentifier()}? sql11ReservedKeywordsUsedAsIdentifier -> Identifier[$sql11ReservedKeywordsUsedAsIdentifier.text] ; functionIdentifier @@ -570,7 +587,44 @@ principalIdentifier | QuotedIdentifier ; +//the new version of nonReserved + sql11ReservedKeywordsUsedAsIdentifier = old version of nonReserved nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER | KW_RELOAD + KW_ADD | KW_ADMIN | KW_AFTER | KW_ANALYZE | KW_ARCHIVE | KW_ASC | KW_BEFORE | KW_BUCKET | KW_BUCKETS + | KW_CASCADE | KW_CHANGE | KW_CLUSTER | KW_CLUSTERED | KW_CLUSTERSTATUS | KW_COLLECTION | KW_COLUMNS + | KW_COMMENT | KW_COMPACT | KW_COMPACTIONS | KW_COMPUTE | KW_CONCATENATE | KW_CONTINUE | KW_DATA + | KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY + | KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_ELEM_TYPE + | KW_ENABLE | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN | KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT + | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOLD_DDLTIME | KW_IDXPROPERTIES | KW_IGNORE + | KW_INDEX | KW_INDEXES | KW_INPATH | KW_INPUTDRIVER | KW_INPUTFORMAT | KW_ITEMS | KW_JAR + | KW_KEYS | KW_KEY_TYPE | KW_LIMIT | KW_LINES | KW_LOAD | KW_LOCATION | KW_LOCK | KW_LOCKS | KW_LOGICAL | KW_LONG + | KW_MAPJOIN | KW_MATERIALIZED | KW_MINUS | KW_MSCK | KW_NOSCAN | KW_NO_DROP | KW_OFFLINE | KW_OPTION + | KW_OUTPUTDRIVER | KW_OUTPUTFORMAT | KW_OVERWRITE | KW_OWNER | KW_PARTITIONED | KW_PARTITIONS | KW_PLUS | KW_PRETTY | KW_PRINCIPALS + | KW_PROTECTION | KW_PURGE | KW_READ | KW_READONLY | KW_REBUILD | KW_RECORDREADER | KW_RECORDWRITER + | KW_REGEXP | KW_RELOAD | KW_RENAME | KW_REPAIR | KW_REPLACE | KW_RESTRICT | KW_REWRITE | KW_RLIKE + | KW_ROLE | KW_ROLES | KW_SCHEMA | KW_SCHEMAS | KW_SEMI | KW_SERDE | KW_SERDEPROPERTIES | KW_SERVER | KW_SETS | KW_SHARED + | KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL | KW_STATISTICS | KW_STORED + | KW_STREAMTABLE | KW_STRING | KW_STRUCT | KW_TABLES | KW_TBLPROPERTIES | KW_TEMPORARY | KW_TERMINATED + | KW_TINYINT | KW_TOUCH | KW_TRANSACTIONS | KW_UNARCHIVE | KW_UNDO | KW_UNIONTYPE | KW_UNLOCK | KW_UNSET + | KW_UNSIGNED | KW_URI | KW_USE | KW_UTC | KW_UTCTIMESTAMP | KW_VALUE_TYPE | KW_VIEW | KW_WHILE + ; + +//The following SQL2011 reserved keywords are used as cast function name only, it is a subset of the sql11ReservedKeywordsUsedAsIdentifier. +sql11ReservedKeywordsUsedAsCastFunctionName + : + KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_DATE | KW_DOUBLE | KW_FLOAT | KW_INT | KW_SMALLINT | KW_TIMESTAMP + ; + +//The following SQL2011 reserved keywords are used as identifiers in many q tests, they may be added back due to backward compatibility. +sql11ReservedKeywordsUsedAsIdentifier + : + KW_ALL | KW_ALTER | KW_ARRAY | KW_AS | KW_AUTHORIZATION | KW_BETWEEN | KW_BIGINT | KW_BINARY | KW_BOOLEAN + | KW_BOTH | KW_BY | KW_CREATE | KW_CUBE | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_CURSOR | KW_DATE | KW_DECIMAL | KW_DELETE | KW_DESCRIBE + | KW_DOUBLE | KW_DROP | KW_EXISTS | KW_EXTERNAL | KW_FALSE | KW_FETCH | KW_FLOAT | KW_FOR | KW_FULL | KW_GRANT + | KW_GROUP | KW_GROUPING | KW_IMPORT | KW_IN | KW_INNER | KW_INSERT | KW_INT | KW_INTERSECT | KW_INTO | KW_IS | KW_LATERAL + | KW_LEFT | KW_LIKE | KW_LOCAL | KW_NONE | KW_NULL | KW_OF | KW_ORDER | KW_OUT | KW_OUTER | KW_PARTITION + | KW_PERCENT | KW_PROCEDURE | KW_RANGE | KW_READS | KW_REVOKE | KW_RIGHT + | KW_ROLLUP | KW_ROW | KW_ROWS | KW_SET | KW_SMALLINT | KW_TABLE | KW_TIMESTAMP | KW_TO | KW_TRIGGER | KW_TRUE + | KW_TRUNCATE | KW_UNION | KW_UPDATE | KW_USER | KW_USING | KW_VALUES | KW_WITH ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java index a24cad9..debd5ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java @@ -193,6 +193,9 @@ public ASTNode parse(String command, Context ctx, boolean setTokenRewriteStream) lexer.setHiveConf(ctx.getConf()); } HiveParser parser = new HiveParser(tokens); + if (ctx != null) { + parser.setHiveConf(ctx.getConf()); + } parser.setTreeAdaptor(adaptor); HiveParser.statement_return r = null; try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g index eba3689..1dcf392 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g @@ -35,6 +35,9 @@ k=3; RecognitionException e) { gParent.errors.add(new ParseError(gParent, e, tokenNames)); } + protected boolean useSQL11ReservedKeywordsForIdentifier() { + return gParent.useSQL11ReservedKeywordsForIdentifier(); + } } @rulecatch { @@ -125,10 +128,11 @@ selectItem @init { gParent.pushMsg("selection target", state); } @after { gParent.popMsg(state); } : + (tableAllColumns) => tableAllColumns -> ^(TOK_SELEXPR tableAllColumns) + | ( expression ((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))? ) -> ^(TOK_SELEXPR expression identifier*) - | tableAllColumns -> ^(TOK_SELEXPR tableAllColumns) ; trfmClause @@ -148,7 +152,9 @@ selectExpression @init { gParent.pushMsg("select expression", state); } @after { gParent.popMsg(state); } : - expression | tableAllColumns + (tableAllColumns) => tableAllColumns + | + expression ; selectExpressionList diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java index eed162b..febf6c5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hive.ql.parse; +import java.io.IOException; + import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Assert; import org.junit.Before; @@ -31,7 +34,6 @@ private static HiveConf conf; private ParseDriver pd; - private SemanticAnalyzer sA; @BeforeClass public static void initialize() { @@ -40,15 +42,20 @@ public static void initialize() { } @Before - public void setup() throws SemanticException { + public void setup() throws SemanticException, IOException { pd = new ParseDriver(); - sA = new CalcitePlanner(conf); } ASTNode parse(String query) throws ParseException { - ASTNode nd = pd.parse(query); + ASTNode nd = null; + try { + nd = pd.parse(query, new Context(conf)); + } catch (IOException e) { + e.printStackTrace(); + } return (ASTNode) nd.getChild(0); } + @Test public void testDeleteNoWhere() throws ParseException { ASTNode ast = parse("DELETE FROM src"); @@ -147,11 +154,11 @@ public void testStandardInsertIntoTable() throws ParseException { @Test public void testSelectStarFromAnonymousVirtTable1Row() throws ParseException { try { - parse("select * from values (3,4)"); + parse("select * from `values` (3,4)"); Assert.assertFalse("Expected ParseException", true); } catch(ParseException ex) { - Assert.assertEquals("Failure didn't match.", "line 1:21 missing EOF at '(' near 'values'",ex.getMessage()); + Assert.assertEquals("Failure didn't match.", "line 1:23 missing EOF at '(' near 'values'",ex.getMessage()); } } @Test diff --git a/ql/src/test/results/clientnegative/select_charliteral.q.out b/ql/src/test/results/clientnegative/select_charliteral.q.out index 09e409b..1d02fa1 100644 --- a/ql/src/test/results/clientnegative/select_charliteral.q.out +++ b/ql/src/test/results/clientnegative/select_charliteral.q.out @@ -1,2 +1 @@ -FAILED: ParseException line 3:11 missing \' at ',' near '_c17' in character string literal -line 4:0 mismatched input '' expecting \' near '_c17' in character string literal +FAILED: ParseException line 3:7 cannot recognize input near '_c17' ',' 'count' in expression specification