diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 433699b..c463dea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -26,12 +26,15 @@ import com.google.common.base.Predicates; import com.google.common.collect.Iterators; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.UnionOperator; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; @@ -72,7 +75,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); opRules.put(new RuleRegExp("R1", "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); - GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null)); + GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, new ProjectRemoverContext(pctx.getConf()))); ArrayList topNodes = new ArrayList(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); @@ -99,6 +102,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // For RS-SEL-RS case. reducer operator in reducer task cannot be null in task compiler return null; } + ProjectRemoverContext ctx = (ProjectRemoverContext) procCtx; + if (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + // if CBO return path is turned on and select is followed by union, + // we turn off this optimization for now. Further investigation is + // needed. + List> children = sel.getChildOperators(); + for (Operator child : children) { + if (child instanceof UnionOperator) { + return null; + } + } + } if(sel.isIdentitySelect()) { parent.removeChildAndAdoptItsChildren(sel); LOG.debug("Identity project remover optimization removed : " + sel); @@ -106,4 +121,19 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } } + public class ProjectRemoverContext implements NodeProcessorCtx { + HiveConf conf; + + public ProjectRemoverContext(HiveConf conf) { + this.conf = conf; + } + + public HiveConf getConf() { + return conf; + } + + public void setConf(HiveConf conf) { + this.conf = conf; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java index 22b4a13..8eb9ee9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java @@ -443,11 +443,25 @@ OpAttr visit(HiveFilter filterRel) throws SemanticException { return inputOpAf.clone(filOp); } + //use this function to make the union "flat" for both execution and explain purpose + private List extractRelNodeFromUnion(HiveUnion unionRel) { + List ret = new ArrayList(); + for (RelNode input : unionRel.getInputs()) { + if (input instanceof HiveUnion) { + ret.addAll(extractRelNodeFromUnion((HiveUnion) input)); + } else { + ret.add(input); + } + } + return ret; + } + OpAttr visit(HiveUnion unionRel) throws SemanticException { // 1. Convert inputs - OpAttr[] inputs = new OpAttr[unionRel.getInputs().size()]; + List inputsList = extractRelNodeFromUnion(unionRel); + OpAttr[] inputs = new OpAttr[inputsList.size()]; for (int i = 0; i < inputs.length; i++) { - inputs[i] = dispatch(unionRel.getInput(i)); + inputs[i] = dispatch(inputsList.get(i)); } if (LOG.isDebugEnabled()) {