diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 67d3a99..e368176 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; +import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; /** * Implementation of the optimizer. @@ -51,6 +52,7 @@ public class Optimizer { transformations.add(new ColumnPruner()); } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { + transformations.add(new PredicateTransitivePropagate()); transformations.add(new PredicatePushDown()); transformations.add(new PartitionPruner()); transformations.add(new PartitionConditionRemover()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java new file mode 100644 index 0000000..f98b661 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class ExprNodeDescUtils { + + public static int indexOf(ExprNodeDesc origin, List sources) { + for (int i = 0; i < sources.size(); i++) { + if (origin.isSame(sources.get(i))) { + return i; + } + } + return -1; + } + + // traversing origin, find ExprNodeDesc in sources and replaces it with ExprNodeDesc + // in targets having same index. + // return null if failed to find + public static ExprNodeDesc replace(ExprNodeDesc origin, + List sources, List targets) { + int index = indexOf(origin, sources); + if (index >= 0) { + return targets.get(index); + } + // encountered column or field which cannot be found in sources + if (origin instanceof ExprNodeColumnDesc || origin instanceof ExprNodeFieldDesc) { + return null; + } + // for ExprNodeGenericFuncDesc, it should be deterministic and stateless + if (origin instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) origin; + if (!FunctionRegistry.isDeterministic(func.getGenericUDF()) + || FunctionRegistry.isStateful(func.getGenericUDF())) { + return null; + } + List children = new ArrayList(); + for (int i = 0; i < origin.getChildren().size(); i++) { + ExprNodeDesc child = replace(origin.getChildren().get(i), sources, targets); + if (child == null) { + return null; + } + children.add(child); + } + // duplicate function with possibily replaced children + ExprNodeGenericFuncDesc clone = (ExprNodeGenericFuncDesc) func.clone(); + clone.setChildExprs(children); + return clone; + } + return origin; + } + + /** + * return true if predicate is already included in source + */ + public static boolean containsPredicate(ExprNodeDesc source, ExprNodeDesc predicate) { + if (source.isSame(predicate)) { + return true; + } + if (FunctionRegistry.isOpAnd(source)) { + if (containsPredicate(source.getChildren().get(0), predicate) || + containsPredicate(source.getChildren().get(1), predicate)) { + return true; + } + } + return false; + } + + /** + * bind two predicates by AND op + */ + public static ExprNodeDesc mergePredicates(ExprNodeDesc prev, ExprNodeDesc next) { + List children = new ArrayList(2); + children.add(prev); + children.add(next); + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, + FunctionRegistry.getGenericUDFForAnd(), children); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index de3d2c6..c74ad20 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; @@ -670,11 +671,10 @@ public final class OpProcFactory { } for (; i < preds.size(); i++) { - List children = new ArrayList(2); - children.add(condn); - children.add(preds.get(i)); - condn = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, - FunctionRegistry.getGenericUDFForAnd(), children); + ExprNodeDesc next = preds.get(i); + if (!ExprNodeDescUtils.containsPredicate(condn, next)) { + condn = ExprNodeDescUtils.mergePredicates(condn, next); + } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java new file mode 100644 index 0000000..b9c2c41 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ppd; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.optimizer.Transform; +import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.JoinCondDesc; +import org.apache.hadoop.hive.ql.plan.JoinDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; + +/** + * propagates filters to other aliases based on join condition + */ +public class PredicateTransitivePropagate implements Transform { + + private ParseContext pGraphContext; + + public ParseContext transform(ParseContext pctx) throws SemanticException { + pGraphContext = pctx; + + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp("R1", "(FIL%RS%JOIN%)|(FIL%RS%MAPJOIN%)"), new JoinTransitive()); + + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); + GraphWalker ogw = new PreOrderWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pGraphContext.getTopOps().values()); + ogw.startWalking(topNodes, null); + + return pGraphContext; + } + + private class JoinTransitive implements NodeProcessor { + + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + @SuppressWarnings("unchecked") + CommonJoinOperator join = (CommonJoinOperator) nd; + + for (int pos : findAxis(join)) { + List reducers = getTargets(join, pos); + if (reducers.isEmpty()) { + return null; + } + reducers.add((ReduceSinkOperator) join.getParentOperators().get(pos)); + + Map, ExprNodeDesc> filters = new HashMap, ExprNodeDesc>(); + for (ReduceSinkOperator source : reducers) { + List sourceKeys = source.getConf().getKeyCols(); + Operator parent = source.getParentOperators().get(0); + if (!(parent instanceof FilterOperator)) { + continue; + } + FilterOperator filter = (FilterOperator) parent; + ExprNodeDesc predicate = filter.getConf().getPredicate(); + for (ReduceSinkOperator target : reducers) { + if (source == target) { + continue; + } + List targetKeys = target.getConf().getKeyCols(); + ExprNodeDesc replaced = ExprNodeDescUtils.replace(predicate, sourceKeys, targetKeys); + if (replaced != null && !filterExists(target, replaced)) { + ExprNodeDesc prev = filters.get(target); + if (prev == null) { + filters.put(target, replaced); + } else { + filters.put(target, ExprNodeDescUtils.mergePredicates(prev, replaced)); + } + } + } + } + // insert filter between RS and parent of RS + for (Map.Entry, ExprNodeDesc> entry : filters.entrySet()) { + Operator target = entry.getKey(); + ExprNodeDesc expr = entry.getValue(); + Operator parent = target.getParentOperators().get(0); + if (parent instanceof FilterOperator) { + ExprNodeDesc prev = ((FilterOperator)parent).getConf().getPredicate(); + ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(prev, expr); + ((FilterOperator)parent).getConf().setPredicate(merged); + } else { + RowResolver parentRR = pGraphContext.getOpParseCtx().get(parent).getRowResolver(); + Operator newFilter = createFilter(target, parent, parentRR, expr); + pGraphContext.getOpParseCtx().put(newFilter, new OpParseContext(parentRR)); + } + } + } + return null; + } + + private Set findAxis(CommonJoinOperator join) { + Set axis = new HashSet(); + for (JoinCondDesc cond : join.getConf().getConds()) { + axis.add(cond.getLeft()); + } + return axis; + } + + private List getTargets(CommonJoinOperator join, int pos) { + List reducers = new ArrayList(); + for (JoinCondDesc cond : join.getConf().getConds()) { + ReduceSinkOperator target = getTarget(join, cond, pos); + if (target != null) { + reducers.add(target); + } + } + return reducers; + } + + // return index of inner joined table with join axis + private ReduceSinkOperator getTarget(Operator join, JoinCondDesc cond, int pos) { + if (cond.getType() != JoinDesc.INNER_JOIN && cond.getType() != JoinDesc.LEFT_SEMI_JOIN) { + // I'm not sure if it's safe to propagate predicates via outer joins + return null; + } + if (cond.getLeft() == pos) { + return (ReduceSinkOperator) join.getParentOperators().get(cond.getRight()); + } + if (cond.getRight() == pos) { + return (ReduceSinkOperator) join.getParentOperators().get(cond.getLeft()); + } + // never + return null; + } + + // insert filter operator between target(chilld) and input(parent) + private Operator createFilter(Operator target, Operator parent, + RowResolver parentRR, ExprNodeDesc filterExpr) { + Operator filter = OperatorFactory.get(new FilterDesc(filterExpr, false), + new RowSchema(parentRR.getColumnInfos())); + filter.setParentOperators(new ArrayList>()); + filter.setChildOperators(new ArrayList>()); + filter.getParentOperators().add(parent); + filter.getChildOperators().add(target); + parent.replaceChild(target, filter); + target.replaceParent(parent, filter); + return filter; + } + + // check same filter exists already + private boolean filterExists(ReduceSinkOperator target, ExprNodeDesc replaced) { + Operator operator = target.getParentOperators().get(0); + for (; operator instanceof FilterOperator; operator = operator.getParentOperators().get(0)) { + ExprNodeDesc predicate = ((FilterOperator) operator).getConf().getPredicate(); + if (ExprNodeDescUtils.containsPredicate(predicate, replaced)) { + return true; + } + } + return false; + } + } +} diff --git ql/src/test/queries/clientpositive/join_nullsafe.q ql/src/test/queries/clientpositive/join_nullsafe.q index 0e3bec5..a566d67 100644 --- ql/src/test/queries/clientpositive/join_nullsafe.q +++ ql/src/test/queries/clientpositive/join_nullsafe.q @@ -55,3 +55,7 @@ SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value <=> b. SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a RIGHT OUTER JOIN smb_input2 b ON a.value <=> b.value ORDER BY a.key, a.value, b.key, b.value; SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value <=> b.value ORDER BY a.key, a.value, b.key, b.value; SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a LEFT OUTER JOIN smb_input2 b ON a.value <=> b.value ORDER BY a.key, a.value, b.key, b.value; + +--HIVE-3315 join predicate transitive +explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL; +select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL; diff --git ql/src/test/results/clientpositive/auto_join11.q.out ql/src/test/results/clientpositive/auto_join11.q.out index 38ec39d..df4b4fc 100644 --- ql/src/test/results/clientpositive/auto_join11.q.out +++ ql/src/test/results/clientpositive/auto_join11.q.out @@ -41,22 +41,26 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {_col1} - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col1} + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + Position of Big Table: 0 Stage: Stage-4 Map Reduce @@ -171,44 +175,48 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {_col1} - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - outputColumnNames: _col0, _col3 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col3 - type: string + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {_col1} + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] outputColumnNames: _col0, _col3 - Group By Operator - aggregations: - expr: sum(hash(_col0,_col3)) - bucketGroup: false - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Position of Big Table: 1 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col3 + type: string + outputColumnNames: _col0, _col3 + Group By Operator + aggregations: + expr: sum(hash(_col0,_col3)) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -242,25 +250,29 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key type: string - tag: 1 - value expressions: - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/auto_join12.q.out ql/src/test/results/clientpositive/auto_join12.q.out index da0b55d..070fa03 100644 --- ql/src/test/results/clientpositive/auto_join12.q.out +++ ql/src/test/results/clientpositive/auto_join12.q.out @@ -52,30 +52,34 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {_col1} - 2 - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - 2 [Column[_col0]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col1} + 2 + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + 2 [Column[_col0]] + Position of Big Table: 0 src3:src TableScan alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: @@ -102,7 +106,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -190,7 +194,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -213,7 +217,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: @@ -238,47 +242,51 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {_col0} - 1 {_col1} - 2 - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - 2 [Column[_col0]] - outputColumnNames: _col0, _col3 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col3 - type: string + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {_col0} + 1 {_col1} + 2 + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + 2 [Column[_col0]] outputColumnNames: _col0, _col3 - Group By Operator - aggregations: - expr: sum(hash(_col0,_col3)) - bucketGroup: false - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Position of Big Table: 1 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col3 + type: string + outputColumnNames: _col0, _col3 + Group By Operator + aggregations: + expr: sum(hash(_col0,_col3)) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -297,7 +305,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -318,24 +326,28 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {_col1} - 2 - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - 2 [Column[_col0]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col1} + 2 + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + 2 [Column[_col0]] + Position of Big Table: 2 Stage: Stage-7 Map Reduce @@ -345,7 +357,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: @@ -397,7 +409,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -419,31 +431,35 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key type: string - tag: 1 - value expressions: - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col1 + type: string src3:src TableScan alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/auto_join13.q.out ql/src/test/results/clientpositive/auto_join13.q.out index 80b08f1..1ed9cf8 100644 --- ql/src/test/results/clientpositive/auto_join13.q.out +++ ql/src/test/results/clientpositive/auto_join13.q.out @@ -53,22 +53,26 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {_col0} {_col1} - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col0} {_col1} + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + Position of Big Table: 0 Stage: Stage-9 Map Reduce @@ -375,31 +379,35 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {_col0} {_col1} - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - outputColumnNames: _col0, _col2, _col3 - Position of Big Table: 1 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {_col0} {_col1} + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + outputColumnNames: _col0, _col2, _col3 + Position of Big Table: 1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -433,27 +441,31 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key type: string - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index 18a1b57..e5b3b61 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -38,15 +38,19 @@ STAGE PLANS: srcpart TableScan alias: srcpart - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + Position of Big Table: 0 Stage: Stage-4 Map Reduce @@ -138,40 +142,44 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - outputColumnNames: _col0, _col5 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col5 - type: string - outputColumnNames: _col0, _col1 + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + outputColumnNames: _col0, _col5 + Position of Big Table: 1 Select Operator expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 + expr: _col0 + type: string + expr: _col5 type: string outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Select Operator + expressions: + expr: UDFToInteger(_col0) + type: int + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work @@ -200,18 +208,22 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: value - type: string + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/auto_join16.q.out ql/src/test/results/clientpositive/auto_join16.q.out index 8983883..5ff5610 100644 --- ql/src/test/results/clientpositive/auto_join16.q.out +++ ql/src/test/results/clientpositive/auto_join16.q.out @@ -43,7 +43,7 @@ STAGE PLANS: alias: tab Filter Operator predicate: - expr: (value < 200.0) + expr: ((key > 20.0) and (value < 200.0)) type: boolean HashTable Sink Operator condition expressions: @@ -174,7 +174,7 @@ STAGE PLANS: alias: tab Filter Operator predicate: - expr: (value < 200.0) + expr: ((key > 20.0) and (value < 200.0)) type: boolean Map Join Operator condition map: @@ -248,7 +248,7 @@ STAGE PLANS: alias: tab Filter Operator predicate: - expr: (value < 200.0) + expr: ((key > 20.0) and (value < 200.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/auto_join20.q.out ql/src/test/results/clientpositive/auto_join20.q.out index 78a18ba..573b283 100644 --- ql/src/test/results/clientpositive/auto_join20.q.out +++ ql/src/test/results/clientpositive/auto_join20.q.out @@ -65,21 +65,25 @@ STAGE PLANS: a:src2 TableScan alias: src2 - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20.0)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-6 Map Reduce @@ -249,20 +253,24 @@ STAGE PLANS: a:src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string a:src3 TableScan alias: src3 @@ -388,7 +396,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key < 15.0)) type: boolean HashTable Sink Operator condition expressions: @@ -410,7 +418,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key < 15.0) + expr: ((key < 15.0) and (key < 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -577,7 +585,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key < 15.0)) type: boolean Reduce Output Operator key expressions: @@ -598,7 +606,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key < 15.0) + expr: ((key < 15.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/auto_join27.q.out ql/src/test/results/clientpositive/auto_join27.q.out index d7a8db6..a2eb8eb 100644 --- ql/src/test/results/clientpositive/auto_join27.q.out +++ ql/src/test/results/clientpositive/auto_join27.q.out @@ -43,35 +43,39 @@ STAGE PLANS: null-subquery2:src_12-subquery2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: key, value - Group By Operator - bucketGroup: false - keys: + Filter Operator + predicate: + expr: (key < 200.0) + type: boolean + Select Operator + expressions: expr: key type: string expr: value type: string - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string - sort order: ++ - Map-reduce partition columns: - expr: _col0 + outputColumnNames: key, value + Group By Operator + bucketGroup: false + keys: + expr: key type: string - expr: _col1 + expr: value type: string - tag: -1 + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + expr: _col1 + type: string + tag: -1 Reduce Operator Tree: Group By Operator bucketGroup: false @@ -161,38 +165,42 @@ STAGE PLANS: null-subquery1:src_12-subquery1:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Union - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Filter Operator + predicate: + expr: (key < 200.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Union + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: + expr: count(1) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -250,23 +258,27 @@ STAGE PLANS: null-subquery1:src_12-subquery1:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Union - HashTable Sink Operator - condition expressions: - 0 - 1 - handleSkewJoin: false - keys: - 0 [Column[_col0]] - 1 [Column[_col0]] - Position of Big Table: 1 + Filter Operator + predicate: + expr: (key < 200.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Union + HashTable Sink Operator + condition expressions: + 0 + 1 + handleSkewJoin: false + keys: + 0 [Column[_col0]] + 1 [Column[_col0]] + Position of Big Table: 1 Stage: Stage-7 Map Reduce @@ -328,23 +340,27 @@ STAGE PLANS: null-subquery1:src_12-subquery1:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Union - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 0 + Filter Operator + predicate: + expr: (key < 200.0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Union + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 0 src3:src TableScan alias: src diff --git ql/src/test/results/clientpositive/auto_join29.q.out ql/src/test/results/clientpositive/auto_join29.q.out index aa80343..9cf8065 100644 --- ql/src/test/results/clientpositive/auto_join29.q.out +++ ql/src/test/results/clientpositive/auto_join29.q.out @@ -6382,7 +6382,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -6422,7 +6422,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Map Join Operator condition map: @@ -6520,7 +6520,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -6560,7 +6560,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean Map Join Operator condition map: @@ -6609,7 +6609,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Reduce Output Operator key expressions: @@ -6630,7 +6630,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: @@ -6749,7 +6749,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -6771,7 +6771,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -6888,7 +6888,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Reduce Output Operator key expressions: @@ -6909,7 +6909,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: @@ -8254,7 +8254,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8272,7 +8272,7 @@ STAGE PLANS: alias: src3 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8294,7 +8294,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Map Join Operator condition map: @@ -8392,7 +8392,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8410,7 +8410,7 @@ STAGE PLANS: alias: src3 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8432,7 +8432,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean Map Join Operator condition map: @@ -8488,7 +8488,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8506,7 +8506,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean HashTable Sink Operator condition expressions: @@ -8528,7 +8528,7 @@ STAGE PLANS: alias: src3 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Map Join Operator condition map: @@ -8577,7 +8577,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Reduce Output Operator key expressions: @@ -8598,7 +8598,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key > 10.0) + expr: ((key > 10.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: @@ -8619,7 +8619,7 @@ STAGE PLANS: alias: src3 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key > 10.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/filter_join_breaktask.q.out ql/src/test/results/clientpositive/filter_join_breaktask.q.out index 674a8bf..78f77d8 100644 --- ql/src/test/results/clientpositive/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/filter_join_breaktask.q.out @@ -67,7 +67,7 @@ STAGE PLANS: Filter Operator isSamplingPred: false predicate: - expr: (value is not null and (value <> '')) + expr: ((key is not null and value is not null) and (value <> '')) type: boolean Reduce Output Operator key expressions: @@ -178,18 +178,23 @@ STAGE PLANS: TableScan alias: g GatherStats: false - Reduce Output Operator - key expressions: - expr: value - type: string - sort order: + - Map-reduce partition columns: - expr: value - type: string - tag: 1 - value expressions: - expr: value - type: string + Filter Operator + isSamplingPred: false + predicate: + expr: (value <> '') + type: boolean + Reduce Output Operator + key expressions: + expr: value + type: string + sort order: + + Map-reduce partition columns: + expr: value + type: string + tag: 1 + value expressions: + expr: value + type: string Needs Tagging: true Path -> Alias: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/join11.q.out ql/src/test/results/clientpositive/join11.q.out index 347a482..a7baa15 100644 --- ql/src/test/results/clientpositive/join11.q.out +++ ql/src/test/results/clientpositive/join11.q.out @@ -52,25 +52,29 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key type: string - tag: 1 - value expressions: - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/join12.q.out ql/src/test/results/clientpositive/join12.q.out index 83c5ff1..62c3ef9 100644 --- ql/src/test/results/clientpositive/join12.q.out +++ ql/src/test/results/clientpositive/join12.q.out @@ -36,7 +36,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -58,31 +58,35 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key type: string - tag: 1 - value expressions: - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col1 + type: string src3:src TableScan alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/join13.q.out ql/src/test/results/clientpositive/join13.q.out index 7044f3a..8cc7950 100644 --- ql/src/test/results/clientpositive/join13.q.out +++ ql/src/test/results/clientpositive/join13.q.out @@ -59,27 +59,31 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 + Filter Operator + predicate: + expr: (key < 100.0) + type: boolean + Select Operator + expressions: + expr: key type: string - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 92e4b7e..c2ee2cb 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -45,18 +45,22 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: value - type: string + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/join16.q.out ql/src/test/results/clientpositive/join16.q.out index 1fc80da..e8ec7e9 100644 --- ql/src/test/results/clientpositive/join16.q.out +++ ql/src/test/results/clientpositive/join16.q.out @@ -48,7 +48,7 @@ STAGE PLANS: alias: tab Filter Operator predicate: - expr: (value < 200.0) + expr: ((key > 20.0) and (value < 200.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/join20.q.out ql/src/test/results/clientpositive/join20.q.out index 29a09a4..a735d36 100644 --- ql/src/test/results/clientpositive/join20.q.out +++ ql/src/test/results/clientpositive/join20.q.out @@ -42,20 +42,24 @@ STAGE PLANS: src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -740,7 +744,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key < 15.0)) type: boolean Reduce Output Operator key expressions: @@ -761,7 +765,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key < 15.0) + expr: ((key < 15.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/join40.q.out ql/src/test/results/clientpositive/join40.q.out index bcd2b59..c6f0d39 100644 --- ql/src/test/results/clientpositive/join40.q.out +++ ql/src/test/results/clientpositive/join40.q.out @@ -1825,20 +1825,24 @@ STAGE PLANS: src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -2523,7 +2527,7 @@ STAGE PLANS: alias: src1 Filter Operator predicate: - expr: (key < 10.0) + expr: ((key < 10.0) and (key < 15.0)) type: boolean Reduce Output Operator key expressions: @@ -2544,7 +2548,7 @@ STAGE PLANS: alias: src2 Filter Operator predicate: - expr: (key < 15.0) + expr: ((key < 15.0) and (key < 10.0)) type: boolean Reduce Output Operator key expressions: diff --git ql/src/test/results/clientpositive/join_nullsafe.q.out ql/src/test/results/clientpositive/join_nullsafe.q.out index d3a2ea4..71e1326 100644 --- ql/src/test/results/clientpositive/join_nullsafe.q.out +++ ql/src/test/results/clientpositive/join_nullsafe.q.out @@ -1588,3 +1588,112 @@ NULL 135 NULL 135 148 NULL 110 NULL 148 NULL 148 NULL 200 200 200 200 +PREHOOK: query: --HIVE-3315 join predicate transitive +explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL +PREHOOK: type: QUERY +POSTHOOK: query: --HIVE-3315 join predicate transitive +explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME myinput1) a) (TOK_TABREF (TOK_TABNAME myinput1) b) (AND (<=> (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) value)) (TOK_FUNCTION TOK_ISNULL (. (TOK_TABLE_OR_COL a) key))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a + TableScan + alias: a + Filter Operator + predicate: + expr: key is null + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: int + sort order: + + Map-reduce partition columns: + expr: key + type: int + tag: 0 + value expressions: + expr: key + type: int + expr: value + type: int + b + TableScan + alias: b + Filter Operator + predicate: + expr: value is null + type: boolean + Reduce Output Operator + key expressions: + expr: value + type: int + sort order: + + Map-reduce partition columns: + expr: value + type: int + tag: 1 + value expressions: + expr: key + type: int + expr: value + type: int + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col1} + 1 {VALUE._col0} {VALUE._col1} + handleSkewJoin: false + nullSafes: [true] + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col1 + type: int + expr: _col4 + type: int + expr: _col5 + type: int + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL +PREHOOK: type: QUERY +PREHOOK: Input: default@myinput1 +#### A masked pattern was here #### +POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL +POSTHOOK: type: QUERY +POSTHOOK: Input: default@myinput1 +#### A masked pattern was here #### +NULL NULL NULL NULL +NULL NULL 10 NULL +NULL NULL 48 NULL +NULL 10 NULL NULL +NULL 10 10 NULL +NULL 10 48 NULL +NULL 35 NULL NULL +NULL 35 10 NULL +NULL 35 48 NULL diff --git ql/src/test/results/clientpositive/ppd_gby_join.q.out ql/src/test/results/clientpositive/ppd_gby_join.q.out index accde64..eb21387 100644 --- ql/src/test/results/clientpositive/ppd_gby_join.q.out +++ ql/src/test/results/clientpositive/ppd_gby_join.q.out @@ -71,7 +71,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key > '2') and (key <> '4')) and (key > '20')) + expr: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) type: boolean Filter Operator predicate: @@ -82,18 +82,22 @@ STAGE PLANS: expr: key type: string outputColumnNames: _col0 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string Reduce Operator Tree: Join Operator condition map: @@ -238,7 +242,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key > '2') and (key <> '4')) and (key > '20')) + expr: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/ppd_join.q.out ql/src/test/results/clientpositive/ppd_join.q.out index dc95030..2dc4486 100644 --- ql/src/test/results/clientpositive/ppd_join.q.out +++ ql/src/test/results/clientpositive/ppd_join.q.out @@ -68,7 +68,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key > '2') and (key <> '4')) and (key > '20')) + expr: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) type: boolean Filter Operator predicate: @@ -81,20 +81,24 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: @@ -627,7 +631,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key > '2') and (key <> '4')) and (key > '20')) + expr: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/ppd_join2.q.out ql/src/test/results/clientpositive/ppd_join2.q.out index 0b275cd..aabd394 100644 --- ql/src/test/results/clientpositive/ppd_join2.q.out +++ ql/src/test/results/clientpositive/ppd_join2.q.out @@ -75,7 +75,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key <> '305') and (key <> '14')) and (key <> '311')) + expr: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) type: boolean Filter Operator predicate: @@ -88,20 +88,24 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: @@ -1815,7 +1819,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (((key <> '305') and (key <> '14')) and (key <> '311')) + expr: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/ppd_join3.q.out ql/src/test/results/clientpositive/ppd_join3.q.out index 3db2486..68bc1dd 100644 --- ql/src/test/results/clientpositive/ppd_join3.q.out +++ ql/src/test/results/clientpositive/ppd_join3.q.out @@ -74,7 +74,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: ((((key <> '12') and (key <> '4')) and (key > '0')) and (key <> '1')) + expr: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) type: boolean Filter Operator predicate: @@ -87,26 +87,30 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string src3:src TableScan alias: src Filter Operator predicate: - expr: ((((key <> '13') and (key <> '1')) and (key > '0')) and (key <> '4')) + expr: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) type: boolean Filter Operator predicate: @@ -117,18 +121,22 @@ STAGE PLANS: expr: key type: string outputColumnNames: _col0 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 2 - value expressions: - expr: _col0 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 2 + value expressions: + expr: _col0 + type: string Reduce Operator Tree: Join Operator condition map: @@ -1832,7 +1840,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: ((((key <> '12') and (key <> '4')) and (key > '0')) and (key <> '1')) + expr: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) type: boolean Select Operator expressions: @@ -1860,7 +1868,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: ((((key <> '13') and (key <> '1')) and (key > '0')) and (key <> '4')) + expr: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) type: boolean Select Operator expressions: diff --git ql/src/test/results/clientpositive/regex_col.q.out ql/src/test/results/clientpositive/regex_col.q.out index d5d74b6..0352853 100644 --- ql/src/test/results/clientpositive/regex_col.q.out +++ ql/src/test/results/clientpositive/regex_col.q.out @@ -251,28 +251,32 @@ STAGE PLANS: b TableScan alias: b - Reduce Output Operator - key expressions: - expr: key - type: string - expr: hr - type: string - expr: ds - type: string - sort order: +++ - Map-reduce partition columns: - expr: key - type: string - expr: hr - type: string - expr: ds - type: string - tag: 1 - value expressions: - expr: ds - type: string - expr: hr - type: string + Filter Operator + predicate: + expr: (key = 103.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + expr: hr + type: string + expr: ds + type: string + sort order: +++ + Map-reduce partition columns: + expr: key + type: string + expr: hr + type: string + expr: ds + type: string + tag: 1 + value expressions: + expr: ds + type: string + expr: hr + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/skewjoin.q.out ql/src/test/results/clientpositive/skewjoin.q.out index 3797b9d..4f46144 100644 --- ql/src/test/results/clientpositive/skewjoin.q.out +++ ql/src/test/results/clientpositive/skewjoin.q.out @@ -1191,7 +1191,7 @@ STAGE PLANS: alias: src Filter Operator predicate: - expr: (key < 100.0) + expr: ((key < 100.0) and (key < 80.0)) type: boolean Select Operator expressions: @@ -1213,31 +1213,35 @@ STAGE PLANS: src2:src TableScan alias: src - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 + Filter Operator + predicate: + expr: ((key < 80.0) and (key < 100.0)) + type: boolean + Select Operator + expressions: + expr: key type: string - tag: 1 - value expressions: - expr: _col1 + expr: value type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col1 + type: string src3:src TableScan alias: src Filter Operator predicate: - expr: (key < 80.0) + expr: ((key < 80.0) and (key < 100.0)) type: boolean Select Operator expressions: