diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index dbd021b..8566374 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -424,15 +424,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } return null; } + logExpr(nd, ewi); + owi.putPrunedPreds((Operator) nd, ewi); if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { // add this filter for deletion, if it does not have non-final candidates - if (ewi.getNonFinalCandidates().values().isEmpty()) { - owi.addCandidateFilterOp((FilterOperator)op); - } + owi.addCandidateFilterOp((FilterOperator)op); + Map> residual = ewi.getResidualPredicates(true); + createFilter(op, residual, owi); } - logExpr(nd, ewi); - owi.putPrunedPreds((Operator) nd, ewi); } // merge it with children predicates boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null); diff --git ql/src/test/queries/clientpositive/join44.q ql/src/test/queries/clientpositive/join44.q new file mode 100644 index 0000000..1de1290 --- /dev/null +++ ql/src/test/queries/clientpositive/join44.q @@ -0,0 +1,13 @@ +set hive.cbo.enable=false; +set hive.optimize.constant.propagation=false; + +-- SORT_QUERY_RESULTS + +CREATE TABLE mytable(val1 INT, val2 INT, val3 INT); + +EXPLAIN +SELECT * +FROM mytable src1, mytable src2 +WHERE src1.val1=src2.val1 + AND src1.val2 between 2450816 and 2451500 + AND src2.val2 between 2450816 and 2451500; diff --git ql/src/test/results/clientpositive/join44.q.out ql/src/test/results/clientpositive/join44.q.out new file mode 100644 index 0000000..8598701 --- /dev/null +++ ql/src/test/results/clientpositive/join44.q.out @@ -0,0 +1,88 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE mytable(val1 INT, val2 INT, val3 INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mytable +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE mytable(val1 INT, val2 INT, val3 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mytable +PREHOOK: query: EXPLAIN +SELECT * +FROM mytable src1, mytable src2 +WHERE src1.val1=src2.val1 + AND src1.val2 between 2450816 and 2451500 + AND src2.val2 between 2450816 and 2451500 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM mytable src1, mytable src2 +WHERE src1.val1=src2.val1 + AND src1.val2 between 2450816 and 2451500 + AND src2.val2 between 2450816 and 2451500 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (val1 is not null and val2 BETWEEN 2450816 AND 2451500) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: val2 (type: int), val3 (type: int) + TableScan + alias: src2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (val1 is not null and val2 BETWEEN 2450816 AND 2451500) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: val2 (type: int), val3 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 val1 (type: int) + 1 val1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (_col0 = _col6) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out index 597b75f..c1c2105 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out @@ -255,8 +255,8 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col12 + _col0) = _col0) (type: boolean) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -272,7 +272,7 @@ STAGE PLANS: key expressions: _col13 (type: string) sort order: + Map-reduce partition columns: _col13 (type: string) - Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) TableScan alias: p3 @@ -294,14 +294,14 @@ STAGE PLANS: 0 _col13 (type: string) 1 p3_name (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32 - Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out index 9b2da59..b0258b8 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out @@ -118,7 +118,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32 Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col1 = _col13) and (_col13 = _col25)) (type: boolean) + predicate: ((_col13 = _col25) and (_col1 = _col13)) (type: boolean) Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string) @@ -202,7 +202,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32 Statistics: Num rows: 28 Data size: 3460 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col13 = _col1) and (_col25 = _col13)) (type: boolean) + predicate: ((_col25 = _col13) and (_col13 = _col1)) (type: boolean) Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string) @@ -267,8 +267,8 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col12 + _col0) = _col0) and _col13 is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col12 + _col0) = _col0) (type: boolean) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -284,7 +284,7 @@ STAGE PLANS: key expressions: _col13 (type: string) sort order: + Map-reduce partition columns: _col13 (type: string) - Statistics: Num rows: 7 Data size: 865 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) TableScan alias: p3 @@ -306,17 +306,17 @@ STAGE PLANS: 0 _col13 (type: string) 1 p3_name (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32 - Statistics: Num rows: 7 Data size: 951 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col12 + _col0) = _col0) and (_col25 = _col13)) (type: boolean) - Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 135 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 380 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out index 6ff13e4..26db67e 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out @@ -282,7 +282,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col13 = _col25) and (_col0 = _col36) and (_col0 = _col12)) (type: boolean) + predicate: (((_col13 = _col25) and (_col0 = _col36)) and (_col0 = _col12)) (type: boolean) Statistics: Num rows: 1 Data size: 123 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string), _col24 (type: int), _col25 (type: string), _col26 (type: string), _col27 (type: string), _col28 (type: string), _col29 (type: int), _col30 (type: string), _col31 (type: double), _col32 (type: string), _col36 (type: int), _col37 (type: string), _col38 (type: string), _col39 (type: string), _col40 (type: string), _col41 (type: int), _col42 (type: string), _col43 (type: double), _col44 (type: string)