Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1397687) +++ conf/hive-default.xml.template (working copy) @@ -928,6 +928,12 @@ + hive.disable.expressions.clusterby + false + If this is true, expressions in cluster by, order by, distribute by and sort by are not allowed. + + + hive.exec.dynamic.partition true Whether or not to allow dynamic partitions in DML/DDL. Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1397687) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -508,6 +508,8 @@ HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", false), // try to use sorted merge bucket map join HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", true), + HIVE_DISABLE_EXPRESSIONS_CLUSTERBY("hive.disable.expressions.clusterby", false), + // optimize skewed join by changing the query plan at compile time HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME("hive.optimize.skewjoin.compiletime", false), Index: ql/src/test/results/clientpositive/expr_disable_check_distributeby1.q.out =================================================================== --- ql/src/test/results/clientpositive/expr_disable_check_distributeby1.q.out (revision 0) +++ ql/src/test/results/clientpositive/expr_disable_check_distributeby1.q.out (working copy) @@ -0,0 +1,54 @@ +PREHOOK: query: -- This conf parameter should not be needed once HIVE-3572 is fixed. +-- expressions in distribute are allowed for some cases like below +-- This needs more investigation, the parameter hive.disable.expressions.clusterby +-- has been added for backward compatibility +explain select key from src distribute by (key + 50) +PREHOOK: type: QUERY +POSTHOOK: query: -- This conf parameter should not be needed once HIVE-3572 is fixed. +-- expressions in distribute are allowed for some cases like below +-- This needs more investigation, the parameter hive.disable.expressions.clusterby +-- has been added for backward compatibility +explain select key from src distribute by (key + 50) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))) (TOK_DISTRIBUTEBY (+ (TOK_TABLE_OR_COL key) 50)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + outputColumnNames: _col0 + Reduce Output Operator + sort order: + Map-reduce partition columns: + expr: (_col0 + 50) + type: double + tag: -1 + value expressions: + expr: _col0 + type: string + Reduce Operator Tree: + Extract + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + Index: ql/src/test/results/clientnegative/expr_disable_check_distributeby1.q.out =================================================================== --- ql/src/test/results/clientnegative/expr_disable_check_distributeby1.q.out (revision 0) +++ ql/src/test/results/clientnegative/expr_disable_check_distributeby1.q.out (working copy) @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10138]: Expressions are not allowed in a distribute by clause. Use a column alias instead Index: ql/src/test/queries/clientpositive/expr_disable_check_distributeby1.q =================================================================== --- ql/src/test/queries/clientpositive/expr_disable_check_distributeby1.q (revision 0) +++ ql/src/test/queries/clientpositive/expr_disable_check_distributeby1.q (working copy) @@ -0,0 +1,7 @@ +set hive.disable.expressions.clusterby = true; + +-- This conf parameter should not be needed once HIVE-3572 is fixed. +-- expressions in distribute are allowed for some cases like below +-- This needs more investigation, the parameter hive.disable.expressions.clusterby +-- has been added for backward compatibility +explain select key from src distribute by (key + 50); Index: ql/src/test/queries/clientnegative/expr_disable_check_distributeby1.q =================================================================== --- ql/src/test/queries/clientnegative/expr_disable_check_distributeby1.q (revision 0) +++ ql/src/test/queries/clientnegative/expr_disable_check_distributeby1.q (working copy) @@ -0,0 +1,7 @@ +set hive.disable.expressions.clusterby = false; + +-- This conf parameter should not be needed once HIVE-3572 is fixed. +-- expressions in distribute used to work for some cases like below +-- This needs more investigation, the parameter hive.disable.expressions.clusterby +-- has been added for backward compatibility +explain select key from src distribute by (key + 50); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1397687) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -6340,6 +6340,14 @@ private void validateExpression(ASTNode expr, ClauseType clauseType) throws SemanticException { + // HIVE-3573 has been filed for this. + // Once that is fixed, this check should go away, we need to be backward compatible with + // what used to work before HIVE-3268. + // Nothing needs to be done if the check is disabled. + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_DISABLE_EXPRESSIONS_CLUSTERBY)) { + return; + } + boolean isGrandChild = true; // The first level of children is whether it is ascending/descending // for order by and sort by