From 54fc8223c85f44918b7574b951139ef4972b7b57 Mon Sep 17 00:00:00 2001 From: Gopal V Date: Mon, 6 Mar 2017 01:44:48 +0530 Subject: [PATCH] with qtest --- .../test/resources/testconfiguration.properties | 1 + .../hive/ql/optimizer/ppr/PartitionPruner.java | 15 ++- .../queries/clientpositive/partition_pruning.q | 12 ++ .../clientpositive/llap/partition_pruning.q.out | 132 +++++++++++++++++++++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 ql/src/test/queries/clientpositive/partition_pruning.q create mode 100644 ql/src/test/results/clientpositive/llap/partition_pruning.q.out diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index b01ebd8..464d760 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -529,6 +529,7 @@ minillaplocal.query.files=acid_globallimit.q,\ orc_ppd_timestamp.q,\ order_null.q,\ partition_multilevels.q,\ + partition_pruning.q,\ ptf.q,\ ptf_streaming.q,\ quotedid_smb.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 655bd8a..64cefa6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDefaultDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.session.SessionState; @@ -67,6 +68,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * The transformation step that does partition pruning. @@ -387,9 +389,20 @@ static private ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List pa referred.add(column); } if (expr instanceof ExprNodeGenericFuncDesc) { + Boolean noop = null; + if (FunctionRegistry.isOpAnd(expr)) { + noop = true; + } else if (FunctionRegistry.isOpOr(expr)) { + noop = false; + } List children = expr.getChildren(); for (int i = 0; i < children.size(); ++i) { - children.set(i, removeNonPartCols(children.get(i), partCols, referred)); + ExprNodeDesc other = removeNonPartCols(children.get(i), partCols, referred); + if (noop != null && ExprNodeDescUtils.isNullConstant(other)) { + Preconditions.checkArgument(expr.getTypeInfo().accept(TypeInfoFactory.booleanTypeInfo)); + other = new ExprNodeConstantDesc(expr.getTypeInfo(), noop); + } + children.set(i, other); } } return expr; diff --git ql/src/test/queries/clientpositive/partition_pruning.q ql/src/test/queries/clientpositive/partition_pruning.q new file mode 100644 index 0000000..159865d --- /dev/null +++ ql/src/test/queries/clientpositive/partition_pruning.q @@ -0,0 +1,12 @@ +create table daysales (customer int) partitioned by (dt string); + +insert into daysales partition(dt='2001-01-01') values(1); +insert into daysales partition(dt='2001-01-03') values(3); + +select * from daysales where nvl(dt='2001-01-01' and customer=1, false); +select * from daysales where nvl(dt='2001-01-02' and customer=1, false); +select * from daysales where nvl(dt='2001-01-01' and customer=1, true); +select * from daysales where (dt='2001-01-01' and customer=1); + + +explain extended select * from daysales where nvl(dt='2001-01-01' and customer=1, false); diff --git ql/src/test/results/clientpositive/llap/partition_pruning.q.out ql/src/test/results/clientpositive/llap/partition_pruning.q.out new file mode 100644 index 0000000..107741c --- /dev/null +++ ql/src/test/results/clientpositive/llap/partition_pruning.q.out @@ -0,0 +1,132 @@ +PREHOOK: query: create table daysales (customer int) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@daysales +POSTHOOK: query: create table daysales (customer int) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@daysales +PREHOOK: query: insert into daysales partition(dt='2001-01-01') values(1) +PREHOOK: type: QUERY +PREHOOK: Output: default@daysales@dt=2001-01-01 +POSTHOOK: query: insert into daysales partition(dt='2001-01-01') values(1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@daysales@dt=2001-01-01 +POSTHOOK: Lineage: daysales PARTITION(dt=2001-01-01).customer EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: insert into daysales partition(dt='2001-01-03') values(3) +PREHOOK: type: QUERY +PREHOOK: Output: default@daysales@dt=2001-01-03 +POSTHOOK: query: insert into daysales partition(dt='2001-01-03') values(3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@daysales@dt=2001-01-03 +POSTHOOK: Lineage: daysales PARTITION(dt=2001-01-03).customer EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from daysales where nvl(dt='2001-01-01' and customer=1, false) +PREHOOK: type: QUERY +PREHOOK: Input: default@daysales +PREHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +POSTHOOK: query: select * from daysales where nvl(dt='2001-01-01' and customer=1, false) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@daysales +POSTHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +1 2001-01-01 +PREHOOK: query: select * from daysales where nvl(dt='2001-01-02' and customer=1, false) +PREHOOK: type: QUERY +PREHOOK: Input: default@daysales +#### A masked pattern was here #### +POSTHOOK: query: select * from daysales where nvl(dt='2001-01-02' and customer=1, false) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@daysales +#### A masked pattern was here #### +PREHOOK: query: select * from daysales where nvl(dt='2001-01-01' and customer=1, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@daysales +PREHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +POSTHOOK: query: select * from daysales where nvl(dt='2001-01-01' and customer=1, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@daysales +POSTHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +1 2001-01-01 +PREHOOK: query: select * from daysales where (dt='2001-01-01' and customer=1) +PREHOOK: type: QUERY +PREHOOK: Input: default@daysales +PREHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +POSTHOOK: query: select * from daysales where (dt='2001-01-01' and customer=1) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@daysales +POSTHOOK: Input: default@daysales@dt=2001-01-01 +#### A masked pattern was here #### +1 2001-01-01 +PREHOOK: query: explain extended select * from daysales where nvl(dt='2001-01-01' and customer=1, false) +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select * from daysales where nvl(dt='2001-01-01' and customer=1, false) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 2001-01-01 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns customer + columns.comments + columns.types int +#### A masked pattern was here #### + name default.daysales + numFiles 1 + numRows 1 + partition_columns dt + partition_columns.types string + rawDataSize 1 + serialization.ddl struct daysales { i32 customer} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns customer + columns.comments + columns.types int +#### A masked pattern was here #### + name default.daysales + partition_columns dt + partition_columns.types string + serialization.ddl struct daysales { i32 customer} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.daysales + name: default.daysales + Processor Tree: + TableScan + alias: daysales + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: NVL((customer = 1),false) (type: boolean) + Select Operator + expressions: customer (type: int), dt (type: string) + outputColumnNames: _col0, _col1 + ListSink + -- 2.4.0