diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 94a5037..07f9ae1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; +import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; /** * Implementation of the optimizer. @@ -52,6 +53,7 @@ public class Optimizer { } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { transformations.add(new PredicatePushDown()); + transformations.add(new PredicateTransitivePropagate()); transformations.add(new PartitionPruner()); transformations.add(new PartitionConditionRemover()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java new file mode 100644 index 0000000..162f7ee --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; + +public class ExprNodeDescUtils { + + public static int indexOf(ExprNodeDesc origin, List sources) { + for (int i = 0; i < sources.size(); i++) { + if (origin.isSame(sources.get(i))) { + return i; + } + } + return -1; + } + + // traversing origin, find ExprNodeDesc in sources and replaces it with ExprNodeDesc + // in targets having same index. + // return null if failed to find + public static ExprNodeDesc replace(ExprNodeDesc origin, + List sources, List targets) { + int index = indexOf(origin, sources); + if (index >= 0) { + return targets.get(index); + } + // encountered column or field which cannot be found in sources + if (origin instanceof ExprNodeColumnDesc || origin instanceof ExprNodeFieldDesc) { + return null; + } + // for ExprNodeGenericFuncDesc, it should be deterministic and stateless + if (origin instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) origin; + if (!FunctionRegistry.isDeterministic(func.getGenericUDF()) + || FunctionRegistry.isStateful(func.getGenericUDF())) { + return null; + } + List children = new ArrayList(); + for (int i = 0; i < origin.getChildren().size(); i++) { + ExprNodeDesc child = replace(origin.getChildren().get(i), sources, targets); + if (child == null) { + return null; + } + children.add(child); + } + // duplicate function with possibily replaced children + ExprNodeGenericFuncDesc clone = (ExprNodeGenericFuncDesc) func.clone(); + clone.setChildExprs(children); + return clone; + } + return origin; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java new file mode 100644 index 0000000..576d75f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ppd; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.optimizer.Transform; +import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.JoinCondDesc; +import org.apache.hadoop.hive.ql.plan.JoinDesc; + +/** + * propagates filters to other aliases based on join condition + */ +public class PredicateTransitivePropagate implements Transform { + + private ParseContext pGraphContext; + + public ParseContext transform(ParseContext pctx) throws SemanticException { + pGraphContext = pctx; + + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp("R1", "(FIL%RS%JOIN%)|(FIL%RS%MAPJOIN%)"), new JoinTransitive()); + + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); + GraphWalker ogw = new PreOrderWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pGraphContext.getTopOps().values()); + ogw.startWalking(topNodes, null); + + return pGraphContext; + } + + private class JoinTransitive implements NodeProcessor { + + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + @SuppressWarnings("unchecked") + CommonJoinOperator join = (CommonJoinOperator) nd; + ReduceSinkOperator source = (ReduceSinkOperator) (join.getParentOperators().get(0)); + int pos = join.getParentOperators().indexOf(source); + + List sourceExprs = source.getConf().getKeyCols(); + Operator current = source.getParentOperators().get(0); + for (; current instanceof FilterOperator; current = current.getParentOperators().get(0)) { + FilterOperator filter = (FilterOperator) current; + ExprNodeDesc predicate = filter.getConf().getPredicate(); + for (JoinCondDesc cond : join.getConf().getConds()) { + ReduceSinkOperator target = getTarget(join, cond, pos); + if (target == null) { + continue; + } + List targetExprs = target.getConf().getKeyCols(); + ExprNodeDesc replaced = ExprNodeDescUtils.replace(predicate, sourceExprs, targetExprs); + if (replaced != null && !filterExists(target, replaced)) { + Operator input = target.getParentOperators().get(0); + RowResolver inputRR = pGraphContext.getOpParseCtx().get(input).getRowResolver(); + Operator newFilter = createFilter(target, input, inputRR, replaced); + pGraphContext.getOpParseCtx().put(newFilter, new OpParseContext(inputRR)); + } + } + } + return null; + } + + private ReduceSinkOperator getTarget(Operator join, JoinCondDesc cond, int pos) { + if (cond.getType() != JoinDesc.INNER_JOIN && cond.getType() != JoinDesc.LEFT_SEMI_JOIN) { + return null; + } + if (cond.getLeft() == pos) { + return (ReduceSinkOperator) join.getParentOperators().get(cond.getRight()); + } + if (cond.getRight() == pos) { + return (ReduceSinkOperator) join.getParentOperators().get(cond.getLeft()); + } + return null; + } + + private Operator createFilter(Operator target, Operator input, + RowResolver inputRR, ExprNodeDesc filterExpr) { + Operator filter = OperatorFactory.get(new FilterDesc(filterExpr, false), + new RowSchema(inputRR.getColumnInfos())); + filter.setParentOperators(new ArrayList>()); + filter.setChildOperators(new ArrayList>()); + filter.getParentOperators().add(input); + filter.getChildOperators().add(target); + input.replaceChild(target, filter); + target.replaceParent(input, filter); + return filter; + } + + // check same filter exists already + private boolean filterExists(ReduceSinkOperator target, ExprNodeDesc replaced) { + Operator operator = target.getParentOperators().get(0); + for (; operator instanceof FilterOperator; operator = operator.getParentOperators().get(0)) { + if (((FilterOperator) operator).getConf().getPredicate().isSame(replaced)) { + return true; + } + } + return false; + } + } +} diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index 18a1b57..e5b3b61 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -38,15 +38,19 @@ STAGE PLANS: srcpart TableScan alias: srcpart - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + Position of Big Table: 0 Stage: Stage-4 Map Reduce @@ -138,40 +142,44 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - outputColumnNames: _col0, _col5 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col5 - type: string - outputColumnNames: _col0, _col1 + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + outputColumnNames: _col0, _col5 + Position of Big Table: 1 Select Operator expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 + expr: _col0 + type: string + expr: _col5 type: string outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Select Operator + expressions: + expr: UDFToInteger(_col0) + type: int + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work @@ -200,18 +208,22 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: value - type: string + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/auto_join20.q.out ql/src/test/results/clientpositive/auto_join20.q.out index 78a18ba..a04199e 100644 --- ql/src/test/results/clientpositive/auto_join20.q.out +++ ql/src/test/results/clientpositive/auto_join20.q.out @@ -65,21 +65,25 @@ STAGE PLANS: a:src2 TableScan alias: src2 - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20.0)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-6 Map Reduce @@ -249,20 +253,24 @@ STAGE PLANS: a:src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string a:src3 TableScan alias: src3 @@ -412,21 +420,25 @@ STAGE PLANS: predicate: expr: (key < 15.0) type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 20.0)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 20.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-6 Map Reduce @@ -600,20 +612,24 @@ STAGE PLANS: predicate: expr: (key < 15.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string a:src3 TableScan alias: src3 diff --git ql/src/test/results/clientpositive/auto_join29.q.out ql/src/test/results/clientpositive/auto_join29.q.out index aa80343..5e99f49 100644 --- ql/src/test/results/clientpositive/auto_join29.q.out +++ ql/src/test/results/clientpositive/auto_join29.q.out @@ -6384,17 +6384,21 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 0 src3 TableScan alias: src3 @@ -6562,42 +6566,46 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - Map Join Operator - condition map: - Inner Join 0 to 1 - Left Outer Join1 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col8 - type: string - expr: _col9 - type: string - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Map Join Operator + condition map: + Inner Join 0 to 1 + Left Outer Join1 to 2 + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Position of Big Table: 1 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col4 + type: string + expr: _col5 + type: string + expr: _col8 + type: string + expr: _col9 + type: string + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -6632,20 +6640,24 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -6773,21 +6785,25 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - filter predicates: - 0 - 1 - 2 {(key < 10.0)} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 + 2 {(key < 10.0)} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-5 Map Reduce @@ -6911,20 +6927,24 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -8256,17 +8276,21 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 0 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 0 src3 TableScan alias: src3 @@ -8434,42 +8458,46 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Position of Big Table: 1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col8 - type: string - expr: _col9 - type: string - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Position of Big Table: 1 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col4 + type: string + expr: _col5 + type: string + expr: _col8 + type: string + expr: _col9 + type: string + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Local Work: Map Reduce Local Work @@ -8508,17 +8536,21 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - 2 [Column[key]] - Position of Big Table: 2 + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + 2 [Column[key]] + Position of Big Table: 2 Stage: Stage-7 Map Reduce @@ -8600,20 +8632,24 @@ STAGE PLANS: predicate: expr: (key > 10.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 diff --git ql/src/test/results/clientpositive/index_auto_mult_tables.q.out ql/src/test/results/clientpositive/index_auto_mult_tables.q.out index 4d885b4..6908ef5 100644 --- ql/src/test/results/clientpositive/index_auto_mult_tables.q.out +++ ql/src/test/results/clientpositive/index_auto_mult_tables.q.out @@ -48,18 +48,22 @@ STAGE PLANS: predicate: expr: ((((key > 70.0) and (key < 90.0)) and (key > 80.0)) and (key < 100.0)) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string + Filter Operator + predicate: + expr: ((((key > 80.0) and (key < 100.0)) and (key > 70.0)) and (key < 90.0)) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string Reduce Operator Tree: Join Operator condition map: @@ -379,18 +383,22 @@ STAGE PLANS: predicate: expr: ((((key > 70.0) and (key < 90.0)) and (key > 80.0)) and (key < 100.0)) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string + Filter Operator + predicate: + expr: ((((key > 80.0) and (key < 100.0)) and (key > 70.0)) and (key < 90.0)) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out index fbfbe03..14ddfed 100644 --- ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out +++ ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out @@ -48,18 +48,22 @@ STAGE PLANS: predicate: expr: ((((key > 70.0) and (key < 90.0)) and (key > 80.0)) and (key < 100.0)) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string + Filter Operator + predicate: + expr: ((((key > 80.0) and (key < 100.0)) and (key > 70.0)) and (key < 90.0)) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string Reduce Operator Tree: Join Operator condition map: @@ -340,18 +344,22 @@ STAGE PLANS: predicate: expr: ((((key > 70.0) and (key < 90.0)) and (key > 80.0)) and (key < 100.0)) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string + Filter Operator + predicate: + expr: ((((key > 80.0) and (key < 100.0)) and (key > 70.0)) and (key < 90.0)) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 92e4b7e..c2ee2cb 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -45,18 +45,22 @@ STAGE PLANS: srcpart TableScan alias: srcpart - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: value - type: string + Filter Operator + predicate: + expr: (key > 100.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/join20.q.out ql/src/test/results/clientpositive/join20.q.out index 29a09a4..203a58d 100644 --- ql/src/test/results/clientpositive/join20.q.out +++ ql/src/test/results/clientpositive/join20.q.out @@ -42,20 +42,24 @@ STAGE PLANS: src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -763,20 +767,24 @@ STAGE PLANS: predicate: expr: (key < 15.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 diff --git ql/src/test/results/clientpositive/join40.q.out ql/src/test/results/clientpositive/join40.q.out index bcd2b59..2acb17d 100644 --- ql/src/test/results/clientpositive/join40.q.out +++ ql/src/test/results/clientpositive/join40.q.out @@ -1825,20 +1825,24 @@ STAGE PLANS: src2 TableScan alias: src2 - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 @@ -2546,20 +2550,24 @@ STAGE PLANS: predicate: expr: (key < 15.0) type: boolean - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string + Filter Operator + predicate: + expr: (key < 10.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + expr: value + type: string src3 TableScan alias: src3 diff --git ql/src/test/results/clientpositive/ppd_gby_join.q.out ql/src/test/results/clientpositive/ppd_gby_join.q.out index accde64..c703ba6 100644 --- ql/src/test/results/clientpositive/ppd_gby_join.q.out +++ ql/src/test/results/clientpositive/ppd_gby_join.q.out @@ -82,18 +82,22 @@ STAGE PLANS: expr: key type: string outputColumnNames: _col0 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/ppd_join.q.out ql/src/test/results/clientpositive/ppd_join.q.out index dc95030..aff08fd 100644 --- ql/src/test/results/clientpositive/ppd_join.q.out +++ ql/src/test/results/clientpositive/ppd_join.q.out @@ -81,20 +81,24 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/ppd_join2.q.out ql/src/test/results/clientpositive/ppd_join2.q.out index 0b275cd..1d5cbac 100644 --- ql/src/test/results/clientpositive/ppd_join2.q.out +++ ql/src/test/results/clientpositive/ppd_join2.q.out @@ -88,20 +88,24 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/ppd_join3.q.out ql/src/test/results/clientpositive/ppd_join3.q.out index 3db2486..5434215 100644 --- ql/src/test/results/clientpositive/ppd_join3.q.out +++ ql/src/test/results/clientpositive/ppd_join3.q.out @@ -87,20 +87,24 @@ STAGE PLANS: expr: value type: string outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string src3:src TableScan alias: src @@ -117,18 +121,22 @@ STAGE PLANS: expr: key type: string outputColumnNames: _col0 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 2 - value expressions: - expr: _col0 - type: string + Filter Operator + predicate: + expr: (_col0 < '400') + type: boolean + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 2 + value expressions: + expr: _col0 + type: string Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/regex_col.q.out ql/src/test/results/clientpositive/regex_col.q.out index b0f8bf2..400bb53 100644 --- ql/src/test/results/clientpositive/regex_col.q.out +++ ql/src/test/results/clientpositive/regex_col.q.out @@ -236,28 +236,32 @@ STAGE PLANS: b TableScan alias: b - Reduce Output Operator - key expressions: - expr: key - type: string - expr: hr - type: string - expr: ds - type: string - sort order: +++ - Map-reduce partition columns: - expr: key - type: string - expr: hr - type: string - expr: ds - type: string - tag: 1 - value expressions: - expr: ds - type: string - expr: hr - type: string + Filter Operator + predicate: + expr: (key = 103.0) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + expr: hr + type: string + expr: ds + type: string + sort order: +++ + Map-reduce partition columns: + expr: key + type: string + expr: hr + type: string + expr: ds + type: string + tag: 1 + value expressions: + expr: ds + type: string + expr: hr + type: string Reduce Operator Tree: Join Operator condition map: