From c96021fa551ceeab5115ac3896fe6c206bc65106 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 24 Jun 2015 17:01:39 -0700 Subject: [PATCH] HIVE-11104 : Select operator doesn't propagate constants appearing in expressions --- .../ql/optimizer/ConstantPropagateProcFactory.java | 4 +- ql/src/test/queries/clientpositive/cp_sel.q | 7 +++ ql/src/test/results/clientpositive/cp_sel.q.out | 56 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ql/src/test/queries/clientpositive/cp_sel.q create mode 100644 ql/src/test/results/clientpositive/cp_sel.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index f9df8e5..bc34229 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.udf.UDFType; @@ -942,6 +941,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. } } colList.set(i, newCol); + if (newCol instanceof ExprNodeConstantDesc) { + constants.put(op.getSchema().getSignature().get(i), newCol); + } if (columnExprMap != null) { columnExprMap.put(columnNames.get(i), newCol); } diff --git a/ql/src/test/queries/clientpositive/cp_sel.q b/ql/src/test/queries/clientpositive/cp_sel.q new file mode 100644 index 0000000..186c700 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cp_sel.q @@ -0,0 +1,7 @@ +select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 order by 1 limit 1; +set hive.exec.dynamic.partition.mode=nonstrict; +create table testpartbucket (key string, value string) partitioned by (ds string, hr string) clustered by(key) sorted by(key) into 2 buckets; +insert overwrite table testpartbucket partition(ds,hr) select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11; +select * from testpartbucket limit 3; +drop table testpartbucket; +reset hive.exec.dynamic.partition.mode; diff --git a/ql/src/test/results/clientpositive/cp_sel.q.out b/ql/src/test/results/clientpositive/cp_sel.q.out new file mode 100644 index 0000000..d576556 --- /dev/null +++ b/ql/src/test/results/clientpositive/cp_sel.q.out @@ -0,0 +1,56 @@ +PREHOOK: query: select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 order by 1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 order by 1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +97 val_97 hello world +PREHOOK: query: create table testpartbucket (key string, value string) partitioned by (ds string, hr string) clustered by(key) sorted by(key) into 2 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testpartbucket +POSTHOOK: query: create table testpartbucket (key string, value string) partitioned by (ds string, hr string) clustered by(key) sorted by(key) into 2 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testpartbucket +PREHOOK: query: insert overwrite table testpartbucket partition(ds,hr) select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@testpartbucket +POSTHOOK: query: insert overwrite table testpartbucket partition(ds,hr) select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@testpartbucket@ds=hello/hr=world +POSTHOOK: Lineage: testpartbucket PARTITION(ds=hello,hr=world).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testpartbucket PARTITION(ds=hello,hr=world).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from testpartbucket limit 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@testpartbucket +PREHOOK: Input: default@testpartbucket@ds=hello/hr=world +#### A masked pattern was here #### +POSTHOOK: query: select * from testpartbucket limit 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testpartbucket +POSTHOOK: Input: default@testpartbucket@ds=hello/hr=world +#### A masked pattern was here #### +238 val_238 hello world +86 val_86 hello world +311 val_311 hello world +PREHOOK: query: drop table testpartbucket +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testpartbucket +PREHOOK: Output: default@testpartbucket +POSTHOOK: query: drop table testpartbucket +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testpartbucket +POSTHOOK: Output: default@testpartbucket -- 1.7.12.4 (Apple Git-37)