From bce9a5a61b05174ac6166c4d47b9b01cbb02a76e Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Sat, 2 May 2015 13:17:47 -0700 Subject: [PATCH] HIVE-9644 : CASE comparison operator rotation optimization --- .../ql/optimizer/ConstantPropagateProcFactory.java | 113 ++++- ql/src/test/queries/clientpositive/fold_case.q | 12 + ql/src/test/queries/clientpositive/fold_when.q | 31 ++ ql/src/test/results/clientpositive/fold_case.q.out | 301 +++++++++++++ ql/src/test/results/clientpositive/fold_when.q.out | 480 +++++++++++++++++++++ .../clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out | 14 +- 6 files changed, 942 insertions(+), 9 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/fold_case.q create mode 100644 ql/src/test/queries/clientpositive/fold_when.q create mode 100644 ql/src/test/results/clientpositive/fold_case.q.out create mode 100644 ql/src/test/results/clientpositive/fold_when.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index e9436e5..e0ce9cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; @@ -65,12 +66,18 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; @@ -79,11 +86,14 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantBooleanObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.BooleanWritable; import com.google.common.collect.ImmutableSet; +import com.sun.tools.javap.ConstantWriter; /** * Factory for generating the different node processors used by ConstantPropagate. @@ -199,10 +209,11 @@ public static ExprNodeDesc foldExpr(ExprNodeGenericFuncDesc funcDesc) { * @param op processing operator * @param propagate if true, assignment expressions will be added to constants. * @return fold expression + * @throws UDFArgumentException */ private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map constants, ConstantPropagateProcCtx cppCtx, Operator op, int tag, - boolean propagate) { + boolean propagate) throws UDFArgumentException { if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; @@ -356,7 +367,7 @@ private static ExprNodeColumnDesc getColumnExpr(ExprNodeDesc expr) { return (expr instanceof ExprNodeColumnDesc) ? (ExprNodeColumnDesc)expr : null; } - private static ExprNodeDesc shortcutFunction(GenericUDF udf, List newExprs) { + private static ExprNodeDesc shortcutFunction(GenericUDF udf, List newExprs) throws UDFArgumentException { if (udf instanceof GenericUDFOPAnd) { for (int i = 0; i < 2; i++) { ExprNodeDesc childExpr = newExprs.get(i); @@ -407,9 +418,107 @@ private static ExprNodeDesc shortcutFunction(GenericUDF udf, List } } + if (udf instanceof GenericUDFWhen) { + if (!(newExprs.size() == 2 || newExprs.size() == 3)) { + // In general, when can have unlimited # of branches, + // we currently only handle either 1 or 2 branch. + return null; + } + ExprNodeDesc thenExpr = newExprs.get(1); + if (thenExpr instanceof ExprNodeNullDesc && (newExprs.size() == 2 || newExprs.get(2) instanceof ExprNodeNullDesc)) { + return thenExpr; + } + ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) : + new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(),null); + + ExprNodeDesc whenExpr = newExprs.get(0); + if (whenExpr instanceof ExprNodeConstantDesc) { + Boolean whenVal = (Boolean)((ExprNodeConstantDesc) whenExpr).getValue(); + return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr; + } + + if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) { + ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr; + ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr; + Object thenVal = constThen.getValue(); + Object elseVal = constElse.getValue(); + if (thenVal == null) { + return elseVal == null ? thenExpr : null; + } else if(thenVal.equals(elseVal)){ + return thenExpr; + } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) { + return Boolean.TRUE.equals(thenVal) ? newExprs.get(0) : + ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)); + } else { + return null; + } + } + } + if (udf instanceof GenericUDFCase) { + // HIVE-9644 Attempt to fold expression like : + // where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end); + // where ss_sold_date= '1998-01-01' ; + if (!(newExprs.size() == 3 || newExprs.size() == 4)) { + // In general case can have unlimited # of branches, + // we currently only handle either 1 or 2 branch. + return null; + } + Boolean[] values = new Boolean[2]; // holds constant boolean value of exprs, if exists. + ExprNodeDesc n = newExprs.get(2); + if (n instanceof ExprNodeConstantDesc) { + values[0] = getBoolValOf((ExprNodeConstantDesc)n); + if (null == values[0]) { + //we failed to determine boolean value of this constant as evaluated by Hive. + return null; + } + } else if (n instanceof ExprNodeNullDesc) { + // for folding purposes, null is as good as false. + values[0] = Boolean.FALSE; + } else { + // non-constant expression. + return null; + } + + if (newExprs.size() == 3) { + // if else branch is missing, it is treated as false. + values[1] = Boolean.FALSE; + } else if (newExprs.get(3) instanceof ExprNodeConstantDesc) { + values[1] = getBoolValOf((ExprNodeConstantDesc)newExprs.get(3)); + if (null == values[1]) { + return null; + } + } else if (newExprs.get(3) instanceof ExprNodeNullDesc) { + values[1] = Boolean.FALSE; + } else { + return null; + } + + if ((Boolean.TRUE.equals(values[0]) && Boolean.TRUE.equals(values[1])) || + (Boolean.FALSE.equals(values[0]) && Boolean.FALSE.equals(values[1]))) { + return newExprs.get(2); + } else if (Boolean.TRUE.equals(values[0])) { + return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)); + } else { + return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)); + } + } + return null; } + private static Boolean getBoolValOf(ExprNodeConstantDesc n) throws UDFArgumentException { + List constExprs = new ArrayList(2); + constExprs.add(n); + constExprs.add(new ExprNodeConstantDesc(Boolean.TRUE)); + ObjectInspector oi = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), constExprs).getWritableObjectInspector(); + if (oi instanceof WritableConstantBooleanObjectInspector) { + BooleanWritable bw = ((WritableConstantBooleanObjectInspector)oi).getWritableConstantValue(); + // for folding purposes, null is as good as false. + return null == bw ? Boolean.FALSE : bw.get(); + } else { + return null; + } + } /** * Evaluate column, replace the deterministic columns with constants if possible * diff --git a/ql/src/test/queries/clientpositive/fold_case.q b/ql/src/test/queries/clientpositive/fold_case.q new file mode 100644 index 0000000..3f9e3a3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/fold_case.q @@ -0,0 +1,12 @@ +explain +select count(1) from src where (case key when '238' then true else false end); +explain +select count(1) from src where (case key when '238' then 1=2 else 1=1 end); +explain +select count(1) from src where (case key when '238' then 1=2 else 1=31 end); +explain +select count(1) from src where (case key when '238' then true else 1=1 end); +explain +select count(1) from src where (case key when '238' then 1=1 else 1=null end); +explain +select count(1) from src where (case key when '238' then null else 1=1 end); diff --git a/ql/src/test/queries/clientpositive/fold_when.q b/ql/src/test/queries/clientpositive/fold_when.q new file mode 100644 index 0000000..e827a5c --- /dev/null +++ b/ql/src/test/queries/clientpositive/fold_when.q @@ -0,0 +1,31 @@ +explain +select key from src where ((case when (key = '238') then null end) = 1); +explain +select key from src where ((case when (key = '238') then null else null end) = 1); +explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 1); +explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 2); +explain +select key from src where ((case when (key = '238') then 1 else null end) = 1); +explain +select key from src where ((case when (key = '238') then 1=1 else null=1 end)); +explain +select key from src where ((case when (key = '238') then 1=1 else 2=2 end)); +explain +select key from src where ((case when (key = '238') then 1=3 else 2=1 end)); +explain +select key from src where ((case when (key = '238') then 1=1 else 2=1 end)); +explain +select key from src where ((case when (key = '238') then 1=3 else 1=1 end)); +explain +select key from src where ((case when ('23' = '23') then 1 else 1 end) = 1); +explain +select key from src where ((case when ('2' = '238') then 1 else 2 end) = 2); +explain +select key from src where ((case when (true=null) then 1 else 1 end) = 1); +explain +select key from src where ((case when (key = (case when (key = '238') then '11' else '11' end)) then false else true end)); +explain +select key from src where ((case when (key = (case when (key = '238') then '12' else '11' end)) then 2=2 else true end)); + diff --git a/ql/src/test/results/clientpositive/fold_case.q.out b/ql/src/test/results/clientpositive/fold_case.q.out new file mode 100644 index 0000000..25cc764 --- /dev/null +++ b/ql/src/test/results/clientpositive/fold_case.q.out @@ -0,0 +1,301 @@ +PREHOOK: query: explain +select count(1) from src where (case key when '238' then true else false end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then true else false end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(1) from src where (case key when '238' then 1=2 else 1=1 end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then 1=2 else 1=1 end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key <> '238') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(1) from src where (case key when '238' then 1=2 else 1=31 end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then 1=2 else 1=31 end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(1) from src where (case key when '238' then true else 1=1 end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then true else 1=1 end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(1) from src where (case key when '238' then 1=1 else 1=null end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then 1=1 else 1=null end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(1) from src where (case key when '238' then null else 1=1 end) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(1) from src where (case key when '238' then null else 1=1 end) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key <> '238') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/fold_when.q.out b/ql/src/test/results/clientpositive/fold_when.q.out new file mode 100644 index 0000000..37803e0 --- /dev/null +++ b/ql/src/test/results/clientpositive/fold_when.q.out @@ -0,0 +1,480 @@ +PREHOOK: query: explain +select key from src where ((case when (key = '238') then null end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then null end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then null else null end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then null else null end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 2) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else 1 end) = 2) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else null end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1 else null end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (CASE WHEN ((key = '238')) THEN (1) ELSE (null) END = 1) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else null=1 end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else null=1 end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: CASE WHEN ((key = '238')) THEN (true) ELSE (null) END (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else 2=2 end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else 2=2 end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1=3 else 2=1 end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1=3 else 2=1 end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else 2=1 end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1=1 else 2=1 end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '238' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = '238') then 1=3 else 1=1 end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = '238') then 1=3 else 1=1 end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (not (key = '238')) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when ('23' = '23') then 1 else 1 end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when ('23' = '23') then 1 else 1 end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain +select key from src where ((case when ('2' = '238') then 1 else 2 end) = 2) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when ('2' = '238') then 1 else 2 end) = 2) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain +select key from src where ((case when (true=null) then 1 else 1 end) = 1) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (true=null) then 1 else 1 end) = 1) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = (case when (key = '238') then '11' else '11' end)) then false else true end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = (case when (key = '238') then '11' else '11' end)) then false else true end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (not (key = '11')) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select key from src where ((case when (key = (case when (key = '238') then '12' else '11' end)) then 2=2 else true end)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from src where ((case when (key = (case when (key = '238') then '12' else '11' end)) then 2=2 else true end)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + ListSink + diff --git a/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out b/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out index 6340a75..381e58f 100644 --- a/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out +++ b/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out @@ -3814,22 +3814,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: lineitem_ix - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 12099 Basic stats: PARTIAL Column stats: COMPLETE Select Operator - expressions: CASE (l_orderkey) WHEN (null) THEN (1) ELSE (1) END (type: int) + expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 12099 Basic stats: PARTIAL Column stats: COMPLETE Group By Operator aggregations: count(_col0) keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -3837,10 +3837,10 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -- 1.7.12.4 (Apple Git-37)