Index: ql/src/test/results/clientpositive/ppd_repeated_alias.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_repeated_alias.q.out (revision 0) +++ ql/src/test/results/clientpositive/ppd_repeated_alias.q.out (revision 0) @@ -0,0 +1,329 @@ +PREHOOK: query: drop table pokes +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table pokes +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table pokes2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table pokes2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table pokes (foo int, bar int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table pokes (foo int, bar int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@pokes +PREHOOK: query: create table pokes2 (foo int, bar int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table pokes2 (foo int, bar int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@pokes2 +PREHOOK: query: -- Q1: predicate should not be pushed on the right side of a left outer join +explain +SELECT a.foo as foo1, b.foo as foo2, b.bar +FROM pokes a LEFT OUTER JOIN pokes2 b +ON a.foo=b.foo +WHERE b.bar=3 +PREHOOK: type: QUERY +POSTHOOK: query: -- Q1: predicate should not be pushed on the right side of a left outer join +explain +SELECT a.foo as foo1, b.foo as foo2, b.bar +FROM pokes a LEFT OUTER JOIN pokes2 b +ON a.foo=b.foo +WHERE b.bar=3 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME pokes) a) (TOK_TABREF (TOK_TABNAME pokes2) b) (= (. (TOK_TABLE_OR_COL a) foo) (. (TOK_TABLE_OR_COL b) foo)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) foo) foo1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) foo) foo2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) bar))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL b) bar) 3)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a + TableScan + alias: a + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 0 + value expressions: + expr: foo + type: int + b + TableScan + alias: b + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 1 + value expressions: + expr: foo + type: int + expr: bar + type: int + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {VALUE._col0} {VALUE._col1} + handleSkewJoin: false + outputColumnNames: _col0, _col4, _col5 + Filter Operator + predicate: + expr: (_col5 = 3) + type: boolean + Select Operator + expressions: + expr: _col0 + type: int + expr: _col4 + type: int + expr: _col5 + type: int + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: -- Q2: predicate should not be pushed on the right side of a left outer join +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, b.bar + FROM pokes a LEFT OUTER JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3 +PREHOOK: type: QUERY +POSTHOOK: query: -- Q2: predicate should not be pushed on the right side of a left outer join +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, b.bar + FROM pokes a LEFT OUTER JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME pokes) a) (TOK_TABREF (TOK_TABNAME pokes2) b) (= (. (TOK_TABLE_OR_COL a) foo) (. (TOK_TABLE_OR_COL b) foo)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) foo) foo1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) foo) foo2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) bar))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (. (TOK_TABLE_OR_COL a) bar) 3)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a:a + TableScan + alias: a + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 0 + value expressions: + expr: foo + type: int + a:b + TableScan + alias: b + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 1 + value expressions: + expr: foo + type: int + expr: bar + type: int + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {VALUE._col0} {VALUE._col1} + handleSkewJoin: false + outputColumnNames: _col0, _col4, _col5 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col4 + type: int + expr: _col5 + type: int + outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: + expr: (_col2 = 3) + type: boolean + Select Operator + expressions: + expr: _col0 + type: int + expr: _col1 + type: int + expr: _col2 + type: int + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: -- Q3: predicate should be pushed +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, a.bar + FROM pokes a JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3 +PREHOOK: type: QUERY +POSTHOOK: query: -- Q3: predicate should be pushed +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, a.bar + FROM pokes a JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME pokes) a) (TOK_TABREF (TOK_TABNAME pokes2) b) (= (. (TOK_TABLE_OR_COL a) foo) (. (TOK_TABLE_OR_COL b) foo)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) foo) foo1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) foo) foo2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) bar))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (. (TOK_TABLE_OR_COL a) bar) 3)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a:a + TableScan + alias: a + Filter Operator + predicate: + expr: (bar = 3) + type: boolean + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 0 + value expressions: + expr: foo + type: int + expr: bar + type: int + a:b + TableScan + alias: b + Reduce Output Operator + key expressions: + expr: foo + type: int + sort order: + + Map-reduce partition columns: + expr: foo + type: int + tag: 1 + value expressions: + expr: foo + type: int + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col1} + 1 {VALUE._col0} + handleSkewJoin: false + outputColumnNames: _col0, _col1, _col4 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col4 + type: int + expr: _col1 + type: int + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col1 + type: int + expr: _col2 + type: int + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: drop table pokes +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@pokes +PREHOOK: Output: default@pokes +POSTHOOK: query: drop table pokes +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@pokes +POSTHOOK: Output: default@pokes +PREHOOK: query: drop table pokes2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@pokes2 +PREHOOK: Output: default@pokes2 +POSTHOOK: query: drop table pokes2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@pokes2 +POSTHOOK: Output: default@pokes2 Index: ql/src/test/queries/clientpositive/ppd_repeated_alias.q =================================================================== --- ql/src/test/queries/clientpositive/ppd_repeated_alias.q (revision 0) +++ ql/src/test/queries/clientpositive/ppd_repeated_alias.q (revision 0) @@ -0,0 +1,30 @@ +drop table pokes; +drop table pokes2; +create table pokes (foo int, bar int); +create table pokes2 (foo int, bar int); + +-- Q1: predicate should not be pushed on the right side of a left outer join +explain +SELECT a.foo as foo1, b.foo as foo2, b.bar +FROM pokes a LEFT OUTER JOIN pokes2 b +ON a.foo=b.foo +WHERE b.bar=3; + +-- Q2: predicate should not be pushed on the right side of a left outer join +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, b.bar + FROM pokes a LEFT OUTER JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3; + +-- Q3: predicate should be pushed +explain +SELECT * FROM + (SELECT a.foo as foo1, b.foo as foo2, a.bar + FROM pokes a JOIN pokes2 b + ON a.foo=b.foo) a +WHERE a.bar=3; + +drop table pokes; +drop table pokes2; \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (revision 1155166) +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (working copy) @@ -23,9 +23,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.Stack; -import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,7 +43,6 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.RowResolver; @@ -389,18 +388,33 @@ } for (Entry> e : childPreds .getFinalCandidates().entrySet()) { + ExprWalkerInfo extractPushdownPreds = ExprWalkerProcFactory + .extractPushdownPreds(owi, op, e.getValue()); if (ignoreAliases || aliases == null || aliases.contains(e.getKey()) || e.getKey() == null) { - // e.getKey() (alias) can be null in case of constant expressions. see - // input8.q - ExprWalkerInfo extractPushdownPreds = ExprWalkerProcFactory - .extractPushdownPreds(owi, op, e.getValue()); + // e.getKey() (alias) can be null in case of constant expressions. + // see input8.q if (!extractPushdownPreds.getNonFinalCandidates().isEmpty()) { hasUnpushedPredicates = true; } - ewi.merge(extractPushdownPreds); - logExpr(nd, extractPushdownPreds); } + ExprWalkerInfo newExprs = new ExprWalkerInfo(); + for (Entry> candidate : + extractPushdownPreds.getFinalCandidates().entrySet()) { + // We filter for aliases here instead of before we call + // extractPushdownPreds() to get aliases in the context of the + // current RowResolver instead of in the contexts of the children + // operator(s). (See HIVE-1342) + if (ignoreAliases || aliases == null + || aliases.contains(candidate.getKey()) + || candidate.getKey() == null) { + // candidate.getKey() (alias) can be null in case of constant + // expressions. see input8.q + newExprs.addPushDowns(candidate.getKey(), candidate.getValue()); + } + } + ewi.merge(newExprs); + logExpr(nd, newExprs); } owi.putPrunedPreds((Operator) nd, ewi); return hasUnpushedPredicates;