diff --git a/ql/pom.xml b/ql/pom.xml index 2243f1c..47e9ec0 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -28,7 +28,7 @@ Hive Query Language - 0.9.1-incubating-SNAPSHOT + 0.9.2-incubating-SNAPSHOT .. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index 4bb99d9..a14770e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -242,7 +242,7 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel *

* JoinPredicateInfo:
* 1. preserves the order of conjuctive elements for - * equi-join(m_equiJoinPredicateElements)
+ * equi-join(equiJoinPredicateElements)
* 2. Stores set of projection indexes from left and right child which is part * of equi join keys; the indexes are both in child and Join node schema.
* 3. Keeps a map of projection indexes that are part of join keys to list of @@ -276,19 +276,19 @@ public JoinPredicateInfo(List nonEquiJoinPredicateElement } public List getNonEquiJoinPredicateElements() { - return nonEquiJoinPredicateElements; + return this.nonEquiJoinPredicateElements; } public List getEquiJoinPredicateElements() { - return equiJoinPredicateElements; + return this.equiJoinPredicateElements; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return projsFromRightPartOfJoinKeysInChildSchema; + return this.projsFromRightPartOfJoinKeysInChildSchema; } /** @@ -297,15 +297,15 @@ public JoinPredicateInfo(List nonEquiJoinPredicateElement * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return projsFromRightPartOfJoinKeysInJoinSchema; + return this.projsFromRightPartOfJoinKeysInJoinSchema; } public Map> getMapOfProjIndxToLeafPInfo() { - return mapOfProjIndxInJoinSchemaToLeafPInfo; + return this.mapOfProjIndxInJoinSchemaToLeafPInfo; } public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { @@ -337,7 +337,7 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); // 2.2 Classify leaf predicate as Equi vs Non Equi - if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { + if (jlpi.comparisonType.equals(SqlKind.EQUALS)) { equiLPIList.add(jlpi); } else { nonEquiLPIList.add(jlpi); @@ -399,38 +399,38 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod * of equi join keys; the indexes are both in child and Join node schema.
*/ public static class JoinLeafPredicateInfo { - private final SqlKind m_comparisonType; - private final ImmutableList m_joinKeyExprsFromLeft; - private final ImmutableList m_joinKeyExprsFromRight; - private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; + private final SqlKind comparisonType; + private final ImmutableList joinKeyExprsFromLeft; + private final ImmutableList joinKeyExprsFromRight; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInJoinSchema) { - m_comparisonType = comparisonType; - m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); - m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); - m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + this.comparisonType = comparisonType; + this.joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); + this.joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); + this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); } public List getJoinKeyExprsFromLeft() { - return m_joinKeyExprsFromLeft; + return this.joinKeyExprsFromLeft; } public List getJoinKeyExprsFromRight() { - return m_joinKeyExprsFromRight; + return this.joinKeyExprsFromRight; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } /** @@ -439,15 +439,15 @@ public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsF * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return m_projsFromRightPartOfJoinKeysInChildSchema; + return this.projsFromRightPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return m_projsFromRightPartOfJoinKeysInJoinSchema; + return this.projsFromRightPartOfJoinKeysInJoinSchema; } private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java index da0f7a4..a8fd118 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java @@ -17,128 +17,86 @@ */ package org.apache.hadoop.hive.ql.optimizer.optiq.rules; -import java.util.ArrayList; import java.util.BitSet; import java.util.List; import java.util.ListIterator; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; +import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; import org.eigenbase.rel.FilterRelBase; import org.eigenbase.rel.JoinRelBase; -import org.eigenbase.rel.JoinRelType; -import org.eigenbase.rel.RelNode; +import org.eigenbase.rel.RelFactories; +import org.eigenbase.rel.rules.PushFilterPastJoinRule; import org.eigenbase.relopt.RelOptRule; import org.eigenbase.relopt.RelOptRuleCall; import org.eigenbase.relopt.RelOptRuleOperand; -import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.relopt.RelOptUtil.InputFinder; import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexCall; import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexUtil; import org.eigenbase.sql.SqlKind; -import org.eigenbase.util.Holder; -public abstract class HivePushFilterPastJoinRule extends RelOptRule { +public abstract class HivePushFilterPastJoinRule extends PushFilterPastJoinRule { + - public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterPastJoinRule( - operand(FilterRelBase.class, operand(HiveJoinRel.class, any())), - "HivePushFilterPastJoinRule:filter", true) { - @Override - public void onMatch(RelOptRuleCall call) { - HiveFilterRel filter = call.rel(0); - HiveJoinRel join = call.rel(1); - perform(call, filter, join); - } - }; - - public static final HivePushFilterPastJoinRule JOIN = new HivePushFilterPastJoinRule( - operand(HiveJoinRel.class, any()), "HivePushFilterPastJoinRule:no-filter", false) { - @Override - public void onMatch(RelOptRuleCall call) { - HiveJoinRel join = call.rel(0); - perform(call, null, join); - } - }; + public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterIntoJoinRule(); - /** Whether to try to strengthen join-type. */ - private final boolean smart; + public static final HivePushFilterPastJoinRule JOIN = new HivePushDownJoinConditionRule(); // ~ Constructors ----------------------------------------------------------- /** * Creates a PushFilterPastJoinRule with an explicit root operand. */ - private HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id, boolean smart) { - super(operand, "PushFilterRule: " + id); - this.smart = smart; + protected HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id, + boolean smart, RelFactories.FilterFactory filterFactory, + RelFactories.ProjectFactory projectFactory) { + super(operand, id, smart, filterFactory, projectFactory); } - // ~ Methods ---------------------------------------------------------------- - - protected void perform(RelOptRuleCall call, FilterRelBase filter, - JoinRelBase join) { - final List joinFilters = RelOptUtil.conjunctions(join - .getCondition()); - - /* - * todo: hb 6/26/14 for left SemiJoin we cannot push predicates yet. The - * assertion that num(JoinRel columns) = num(leftSrc) + num(rightSrc) - * doesn't hold. So RelOptUtil.classifyFilters fails. - */ - if (((HiveJoinRel) join).isLeftSemiJoin()) { - return; + /** Rule that tries to push filter expressions into a join + * condition and into the inputs of the join. */ + public static class HivePushFilterIntoJoinRule extends HivePushFilterPastJoinRule { + public HivePushFilterIntoJoinRule() { + super( + RelOptRule.operand(FilterRelBase.class, + RelOptRule.operand(JoinRelBase.class, RelOptRule.any())), + "HivePushFilterPastJoinRule:filter", + true, HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveProjectRel.DEFAULT_PROJECT_FACTORY); } - if (filter == null) { - // There is only the joinRel - // make sure it does not match a cartesian product joinRel - // (with "true" condition) otherwise this rule will be applied - // again on the new cartesian product joinRel. - boolean onlyTrueFilter = true; - for (RexNode joinFilter : joinFilters) { - if (!joinFilter.isAlwaysTrue()) { - onlyTrueFilter = false; - break; - } - } - - if (onlyTrueFilter) { - return; - } + @Override + public void onMatch(RelOptRuleCall call) { + FilterRelBase filter = call.rel(0); + JoinRelBase join = call.rel(1); + super.perform(call, filter, join); } + } - final List aboveFilters = filter != null ? RelOptUtil - .conjunctions(filter.getCondition()) : new ArrayList(); - - List leftFilters = new ArrayList(); - List rightFilters = new ArrayList(); - int origJoinFiltersSz = joinFilters.size(); - - // TODO - add logic to derive additional filters. E.g., from - // (t1.a = 1 AND t2.a = 2) OR (t1.b = 3 AND t2.b = 4), you can - // derive table filters: - // (t1.a = 1 OR t1.b = 3) - // (t2.a = 2 OR t2.b = 4) - - // Try to push down above filters. These are typically where clause - // filters. They can be pushed down if they are not on the NULL - // generating side. - boolean filterPushed = false; - final Holder joinTypeHolder = Holder.of(join.getJoinType()); - if (RelOptUtil.classifyFilters(join, aboveFilters, - join.getJoinType(), true, !join.getJoinType().generatesNullsOnLeft(), !join.getJoinType() - .generatesNullsOnRight(), joinFilters, leftFilters, rightFilters, joinTypeHolder, smart)) { - filterPushed = true; - } + public static class HivePushDownJoinConditionRule + extends HivePushFilterPastJoinRule { +public HivePushDownJoinConditionRule() { + super(RelOptRule.operand(JoinRelBase.class, RelOptRule.any()), + "HivePushFilterPastJoinRule:no-filter", + true, HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveProjectRel.DEFAULT_PROJECT_FACTORY); +} - /* - * Any predicates pushed down to joinFilters that aren't equality - * conditions: put them back as aboveFilters because Hive doesn't support - * not equi join conditions. - */ +@Override +public void onMatch(RelOptRuleCall call) { + JoinRelBase join = call.rel(0); + super.perform(call, null, join); +} +} + + // ~ Methods ---------------------------------------------------------------- + /* + * Any predicates pushed down to joinFilters that aren't equality conditions: + * put them back as aboveFilters because Hive doesn't support not equi join + * conditions. + */ + @Override + protected void validateJoinFilters(List aboveFilters, List joinFilters, + JoinRelBase join) { ListIterator filterIter = joinFilters.listIterator(); while (filterIter.hasNext()) { RexNode exp = filterIter.next(); @@ -162,91 +120,34 @@ protected void perform(RelOptRuleCall call, FilterRelBase filter, aboveFilters.add(exp); filterIter.remove(); } + } - /* - * if all pushed filters where put back then set filterPushed to false - */ - if (leftFilters.size() == 0 && rightFilters.size() == 0 - && joinFilters.size() == origJoinFiltersSz) { - filterPushed = false; - } - - // Try to push down filters in ON clause. A ON clause filter can only be - // pushed down if it does not affect the non-matching set, i.e. it is - // not on the side which is preserved. - if (RelOptUtil.classifyFilters(join, joinFilters, null, false, !join - .getJoinType().generatesNullsOnRight(), !join.getJoinType() - .generatesNullsOnLeft(), joinFilters, leftFilters, rightFilters, joinTypeHolder, false)) { - filterPushed = true; - } - - if (!filterPushed) { - return; - } - - /* - * Remove always true conditions that got pushed down. - */ - removeAlwaysTruePredicates(leftFilters); - removeAlwaysTruePredicates(rightFilters); - removeAlwaysTruePredicates(joinFilters); - - // create FilterRels on top of the children if any filters were - // pushed to them - RexBuilder rexBuilder = join.getCluster().getRexBuilder(); - RelNode leftRel = createFilterOnRel(rexBuilder, join.getLeft(), leftFilters); - RelNode rightRel = createFilterOnRel(rexBuilder, join.getRight(), - rightFilters); + @Override + protected void handleEmptyJoinFilters(List joinFilters, + RexBuilder rexBuilder) { + joinFilters.add(rexBuilder.makeLiteral(true)); + } - // create the new join node referencing the new children and - // containing its new join filters (if there are any) - RexNode joinFilter; + private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { + boolean refersToBothSides = false; - if (joinFilters.size() == 0) { - // if nothing actually got pushed and there is nothing leftover, - // then this rule is a no-op - if (leftFilters.isEmpty() - && rightFilters.isEmpty() - && joinTypeHolder.get() == join.getJoinType()) { - return; - } - joinFilter = rexBuilder.makeLiteral(true); - } else { - joinFilter = RexUtil.composeConjunction(rexBuilder, joinFilters, true); - } - RelNode newJoinRel = HiveJoinRel.getJoin(join.getCluster(), leftRel, - rightRel, joinFilter, join.getJoinType(), false); + int joinNoOfProjects = j.getRowType().getFieldCount(); + BitSet filterProjs = new BitSet(joinNoOfProjects); + BitSet allLeftProjs = new BitSet(joinNoOfProjects); + BitSet allRightProjs = new BitSet(joinNoOfProjects); + allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true); + allRightProjs.set(j.getInput(0).getRowType().getFieldCount(), joinNoOfProjects, true); - // create a FilterRel on top of the join if needed - RelNode newRel = createFilterOnRel(rexBuilder, newJoinRel, aboveFilters); + InputFinder inputFinder = new InputFinder(filterProjs); + filter.accept(inputFinder); - call.transformTo(newRel); - } + if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) + refersToBothSides = true; - /** - * If the filter list passed in is non-empty, creates a FilterRel on top of - * the existing RelNode; otherwise, just returns the RelNode - * - * @param rexBuilder - * rex builder - * @param rel - * the RelNode that the filter will be put on top of - * @param filters - * list of filters - * @return new RelNode or existing one if no filters - */ - private RelNode createFilterOnRel(RexBuilder rexBuilder, RelNode rel, - List filters) { - RexNode andFilters = RexUtil.composeConjunction(rexBuilder, filters, false); - if (andFilters.isAlwaysTrue()) { - return rel; - } - return new HiveFilterRel(rel.getCluster(), rel.getCluster().traitSetOf( - HiveRel.CONVENTION), rel, andFilters); + return refersToBothSides; } private void removeAlwaysTruePredicates(List predicates) { - ListIterator iter = predicates.listIterator(); while (iter.hasNext()) { RexNode exp = iter.next(); @@ -265,26 +166,7 @@ private boolean isAlwaysTrue(RexNode predicate) { } } return predicate.isAlwaysTrue(); - } - - private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { - boolean refersToBothSides = false; - - int joinNoOfProjects = j.getRowType().getFieldCount(); - BitSet filterProjs = new BitSet(joinNoOfProjects); - BitSet allLeftProjs = new BitSet(joinNoOfProjects); - BitSet allRightProjs = new BitSet(joinNoOfProjects); - allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true); - allRightProjs.set(j.getInput(0).getRowType().getFieldCount(), joinNoOfProjects, true); - - InputFinder inputFinder = new InputFinder(filterProjs); - filter.accept(inputFinder); - - if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) - refersToBothSides = true; - - return refersToBothSides; - } + } } // End PushFilterPastJoinRule.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java index 214a29f..ba07363 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java @@ -34,13 +34,13 @@ import org.eigenbase.sql.SqlKind; public class FilterSelectivityEstimator extends RexVisitorImpl { - private final RelNode m_childRel; - private final double m_childCardinality; + private final RelNode childRel; + private final double childCardinality; protected FilterSelectivityEstimator(RelNode childRel) { super(true); - m_childRel = childRel; - m_childCardinality = RelMetadataQuery.getRowCount(m_childRel); + this.childRel = childRel; + this.childCardinality = RelMetadataQuery.getRowCount(childRel); } public Double estimateSelectivity(RexNode predicate) { @@ -53,11 +53,10 @@ public Double visitCall(RexCall call) { } /* - * Ignore any predicates on partition columns - * because we have already accounted for these in - * the Table row count. + * Ignore any predicates on partition columns because we have already + * accounted for these in the Table row count. */ - if (isPartitionPredicate(call, m_childRel)) { + if (isPartitionPredicate(call, this.childRel)) { return 1.0; } @@ -151,10 +150,10 @@ private Double computeDisjunctionSelectivity(RexCall call) { if (tmpSelectivity == null) { tmpSelectivity = 0.99; } - tmpCardinality = m_childCardinality * tmpSelectivity; + tmpCardinality = childCardinality * tmpSelectivity; if (tmpCardinality > 1) - tmpSelectivity = (1 - tmpCardinality / m_childCardinality); + tmpSelectivity = (1 - tmpCardinality / childCardinality); else tmpSelectivity = 1.0; @@ -195,7 +194,7 @@ private Double getMaxNDV(RexCall call) { for (RexNode op : call.getOperands()) { if (op instanceof RexInputRef) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, ((RexInputRef) op).getIndex()); if (tmpNDV > maxNDV) maxNDV = tmpNDV; @@ -203,7 +202,7 @@ private Double getMaxNDV(RexCall call) { irv = new InputReferencedVisitor(); irv.apply(op); for (Integer childProjIndx : irv.inputPosReferenced) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, childProjIndx); + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, childProjIndx); if (tmpNDV > maxNDV) maxNDV = tmpNDV; } @@ -214,14 +213,13 @@ private Double getMaxNDV(RexCall call) { } private boolean isPartitionPredicate(RexNode expr, RelNode r) { - if ( r instanceof ProjectRelBase ) { + if (r instanceof ProjectRelBase) { expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r); return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild()); - } else if ( r instanceof FilterRelBase ) { + } else if (r instanceof FilterRelBase) { return isPartitionPredicate(expr, ((FilterRelBase) r).getChild()); - } else if ( r instanceof HiveTableScanRel ) { - RelOptHiveTable table = (RelOptHiveTable) - ((HiveTableScanRel)r).getTable(); + } else if (r instanceof HiveTableScanRel) { + RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScanRel) r).getTable(); BitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java index ec85603..0242b5e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java @@ -82,35 +82,35 @@ private static final Log LOG = LogFactory.getLog(RexNodeConverter.class); private static class InputCtx { - private final RelDataType m_optiqInpDataType; - private final ImmutableMap m_hiveNameToPosMap; - private final RowResolver m_hiveRR; - private final int m_offsetInOptiqSchema; + private final RelDataType optiqInpDataType; + private final ImmutableMap hiveNameToPosMap; + private final RowResolver hiveRR; + private final int offsetInOptiqSchema; private InputCtx(RelDataType optiqInpDataType, ImmutableMap hiveNameToPosMap, RowResolver hiveRR, int offsetInOptiqSchema) { - m_optiqInpDataType = optiqInpDataType; - m_hiveNameToPosMap = hiveNameToPosMap; - m_hiveRR = hiveRR; - m_offsetInOptiqSchema = offsetInOptiqSchema; + this.optiqInpDataType = optiqInpDataType; + this.hiveNameToPosMap = hiveNameToPosMap; + this.hiveRR = hiveRR; + this.offsetInOptiqSchema = offsetInOptiqSchema; } }; - private final RelOptCluster m_cluster; - private final ImmutableList m_inputCtxs; - private final boolean m_flattenExpr; + private final RelOptCluster cluster; + private final ImmutableList inputCtxs; + private final boolean flattenExpr; public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap nameToPosMap, int offset, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + this.inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); + this.flattenExpr = flattenExpr; } public RexNodeConverter(RelOptCluster cluster, List inpCtxLst, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + this.inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); + this.flattenExpr = flattenExpr; } public RexNode convert(ExprNodeDesc expr) throws SemanticException { @@ -134,7 +134,7 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept RexNode rexNode = convert(fieldDesc.getDesc()); if (rexNode instanceof RexCall) { // regular case of accessing nested field in a column - return m_cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); + return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); } else { // This may happen for schema-less tables, where columns are dynamically // supplied by serdes. @@ -184,7 +184,7 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep } } - argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), m_cluster.getTypeFactory())); + argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory())); tmpRN = convert(tmpExprNode); childRexNodeLst.add(tmpRN); } @@ -196,20 +196,20 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep if (expr == null) { // This is not a cast; process the function. - retType = TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()); + retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()); SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType); - expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); + expr = cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); } else { retType = expr.getType(); } // TODO: Cast Function in Optiq have a bug where it infertype on cast throws // an exception - if (m_flattenExpr && (expr instanceof RexCall) + if (flattenExpr && (expr instanceof RexCall) && !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) { RexCall call = (RexCall) expr; - expr = m_cluster.getRexBuilder().makeCall(retType, call.getOperator(), + expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(), RexUtil.flatten(call.getOperands(), call.getOperator())); } @@ -246,8 +246,8 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar) || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate) || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { - castExpr = m_cluster.getRexBuilder().makeAbstractCast( - TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()), + castExpr = cluster.getRexBuilder().makeAbstractCast( + TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()), childRexNodeLst.get(0)); } } @@ -258,15 +258,15 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { InputCtx ctxLookingFor = null; - if (m_inputCtxs.size() == 1) { - ctxLookingFor = m_inputCtxs.get(0); + if (inputCtxs.size() == 1) { + ctxLookingFor = inputCtxs.get(0); } else { String tableAlias = col.getTabAlias(); String colAlias = col.getColumn(); int noInp = 0; - for (InputCtx ic : m_inputCtxs) { - if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) { - if (ic.m_hiveRR.getPosition(colAlias) >= 0) { + for (InputCtx ic : inputCtxs) { + if (tableAlias == null || ic.hiveRR.hasTableAlias(tableAlias)) { + if (ic.hiveRR.getPosition(colAlias) >= 0) { ctxLookingFor = ic; noInp++; } @@ -282,16 +282,16 @@ private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException { InputCtx ic = getInputCtx(col); - int pos = ic.m_hiveNameToPosMap.get(col.getColumn()); - return m_cluster.getRexBuilder().makeInputRef( - ic.m_optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.m_offsetInOptiqSchema); + int pos = ic.hiveNameToPosMap.get(col.getColumn()); + return cluster.getRexBuilder().makeInputRef( + ic.optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.offsetInOptiqSchema); } private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE), MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticException { - RexBuilder rexBuilder = m_cluster.getRexBuilder(); + RexBuilder rexBuilder = cluster.getRexBuilder(); RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory(); PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo(); RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory); @@ -353,7 +353,7 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce // will work... // An alternative would be to throw CboSemanticException and fall back // to no CBO. - RelDataType relType = m_cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, + RelDataType relType = cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length()); optiqLiteral = rexBuilder.makeExactLiteral(bd, relType); } @@ -397,8 +397,8 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce } private RexNode createNullLiteral(ExprNodeDesc expr) throws OptiqSemanticException { - return m_cluster.getRexBuilder().makeNullLiteral( - TypeConverter.convert(expr.getTypeInfo(), m_cluster.getTypeFactory()).getSqlTypeName()); + return cluster.getRexBuilder().makeNullLiteral( + TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName()); } public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java index 31f906a..793f117 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java @@ -64,7 +64,7 @@ import com.google.common.collect.Maps; public class SqlFunctionConverter { - private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); + private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); static final Map hiveToOptiq; static final Map optiqToHiveToken; @@ -87,10 +87,12 @@ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveU } // do generic lookup String name = null; if (StringUtils.isEmpty(funcTextName)) { - name = getName(hiveUDF); // this should probably never happen, see getName comment + name = getName(hiveUDF); // this should probably never happen, see getName + // comment LOG.warn("The function text was empty, name from annotation is " + name); } else { - // We could just do toLowerCase here and let SA qualify it, but let's be proper... + // We could just do toLowerCase here and let SA qualify it, but let's be + // proper... name = FunctionRegistry.getNormalizedFunctionName(funcTextName); } return getOptiqFn(name, optiqArgTypes, retType); @@ -115,11 +117,10 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { if (castType.equals(TypeInfoFactory.byteTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("tinyint"); } else if (castType instanceof CharTypeInfo) { - castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("char")); + castUDF = handleCastForParameterizedType(castType, FunctionRegistry.getFunctionInfo("char")); } else if (castType instanceof VarcharTypeInfo) { castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("varchar")); + FunctionRegistry.getFunctionInfo("varchar")); } else if (castType.equals(TypeInfoFactory.stringTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("string"); } else if (castType.equals(TypeInfoFactory.booleanTypeInfo)) { @@ -140,24 +141,24 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { castUDF = FunctionRegistry.getFunctionInfo("datetime"); } else if (castType instanceof DecimalTypeInfo) { castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("decimal")); + FunctionRegistry.getFunctionInfo("decimal")); } else if (castType.equals(TypeInfoFactory.binaryTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("binary"); - } else throw new IllegalStateException("Unexpected type : " + - castType.getQualifiedName()); + } else + throw new IllegalStateException("Unexpected type : " + castType.getQualifiedName()); } return castUDF; } private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, FunctionInfo fi) { - SettableUDF udf = (SettableUDF)fi.getGenericUDF(); + SettableUDF udf = (SettableUDF) fi.getGenericUDF(); try { udf.setTypeInfo(ti); } catch (UDFArgumentException e) { throw new RuntimeException(e); } - return new FunctionInfo(fi.isNative(),fi.getDisplayName(),(GenericUDF)udf); + return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf); } // TODO: 1) handle Agg Func Name translation 2) is it correct to add func args @@ -175,11 +176,10 @@ public static ASTNode buildAST(SqlOperator op, List children) { } else if (op.kind == SqlKind.PLUS_PREFIX) { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.PLUS, "PLUS"); } else { - if (op.getName().toUpperCase() - .equals(SqlStdOperatorTable.COUNT.getName()) + if (op.getName().toUpperCase().equals(SqlStdOperatorTable.COUNT.getName()) && children.size() == 0) { - node = (ASTNode) ParseDriver.adaptor.create( - HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); + node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTIONSTAR, + "TOK_FUNCTIONSTAR"); } node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName())); } @@ -210,9 +210,12 @@ public static ASTNode buildAST(SqlOperator op, List children, int i) { } - // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry, - // and only happen to match annotations. For user UDFs, the name is what user specifies at - // creation time (annotation can be absent, different, or duplicate some other function). + // TODO: this is not valid. Function names for built-in UDFs are specified in + // FunctionRegistry, + // and only happen to match annotations. For user UDFs, the name is what user + // specifies at + // creation time (annotation can be absent, different, or duplicate some other + // function). private static String getName(GenericUDF hiveUDF) { String udfName = null; if (hiveUDF instanceof GenericUDFBridge) { @@ -287,70 +290,72 @@ private static HiveToken hToken(int type, String text) { } public static class OptiqUDAF extends SqlAggFunction { - final ImmutableList m_argTypes; - final RelDataType m_retType; + final ImmutableList argTypes; + final RelDataType retType; public OptiqUDAF(String opName, SqlReturnTypeInference returnTypeInference, SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, ImmutableList argTypes, RelDataType retType) { super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference, operandTypeInference, operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); - m_argTypes = argTypes; - m_retType = retType; + this.argTypes = argTypes; + this.retType = retType; } @Override public List getParameterTypes(final RelDataTypeFactory typeFactory) { - return m_argTypes; + return this.argTypes; } @Override public RelDataType getReturnType(final RelDataTypeFactory typeFactory) { - return m_retType; + return this.retType; } } private static class OptiqUDFInfo { - private String m_udfName; - private SqlReturnTypeInference m_returnTypeInference; - private SqlOperandTypeInference m_operandTypeInference; - private SqlOperandTypeChecker m_operandTypeChecker; - private ImmutableList m_argTypes; - private RelDataType m_retType; + private String udfName; + private SqlReturnTypeInference returnTypeInference; + private SqlOperandTypeInference operandTypeInference; + private SqlOperandTypeChecker operandTypeChecker; + private ImmutableList argTypes; + private RelDataType retType; } private static OptiqUDFInfo getUDFInfo(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { OptiqUDFInfo udfInfo = new OptiqUDFInfo(); - udfInfo.m_udfName = hiveUdfName; - udfInfo.m_returnTypeInference = ReturnTypes.explicit(optiqRetType); - udfInfo.m_operandTypeInference = InferTypes.explicit(optiqArgTypes); + udfInfo.udfName = hiveUdfName; + udfInfo.returnTypeInference = ReturnTypes.explicit(optiqRetType); + udfInfo.operandTypeInference = InferTypes.explicit(optiqArgTypes); ImmutableList.Builder typeFamilyBuilder = new ImmutableList.Builder(); for (RelDataType at : optiqArgTypes) { typeFamilyBuilder.add(Util.first(at.getSqlTypeName().getFamily(), SqlTypeFamily.ANY)); } - udfInfo.m_operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); + udfInfo.operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); - udfInfo.m_argTypes = ImmutableList. copyOf(optiqArgTypes); - udfInfo.m_retType = optiqRetType; + udfInfo.argTypes = ImmutableList. copyOf(optiqArgTypes); + udfInfo.retType = optiqRetType; return udfInfo; } public static SqlOperator getOptiqFn(String hiveUdfName, - ImmutableList optiqArgTypes, RelDataType optiqRetType) throws OptiqSemanticException{ + ImmutableList optiqArgTypes, RelDataType optiqRetType) + throws OptiqSemanticException { if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) { // We can create Optiq IS_DISTINCT_FROM operator for this. But since our - // join reordering algo cant handle this anyway there is no advantage of this. + // join reordering algo cant handle this anyway there is no advantage of + // this. // So, bail out for now. throw new OptiqSemanticException("<=> is not yet supported for cbo."); } SqlOperator optiqOp = hiveToOptiq.get(hiveUdfName); if (optiqOp == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqOp = new SqlFunction(uInf.m_udfName, SqlKind.OTHER_FUNCTION, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, + optiqOp = new SqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, + uInf.operandTypeInference, uInf.operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); } @@ -363,8 +368,8 @@ public static SqlAggFunction getOptiqAggFn(String hiveUdfName, if (optiqAggFn == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqAggFn = new OptiqUDAF(uInf.m_udfName, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, uInf.m_argTypes, uInf.m_retType); + optiqAggFn = new OptiqUDAF(uInf.udfName, uInf.returnTypeInference, uInf.operandTypeInference, + uInf.operandTypeChecker, uInf.argTypes, uInf.retType); } return optiqAggFn; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 48d334a..e3f65c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -42,6 +42,7 @@ import java.util.regex.PatternSyntaxException; import com.google.common.annotations.VisibleForTesting; + import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.tools.Frameworks; @@ -274,6 +275,7 @@ import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; +import org.eigenbase.rex.RexUtil; import org.eigenbase.rex.RexWindowBound; import org.eigenbase.rex.RexFieldCollation; import org.eigenbase.sql.SqlAggFunction; @@ -12821,8 +12823,9 @@ private RelNode genFilterRelNode(ASTNode filterExpr, RelNode srcRel) throws Sema .get(srcRel); RexNode convertedFilterExpr = new RexNodeConverter(cluster, srcRel.getRowType(), hiveColNameOptiqPosMap, 0, true).convert(filterCondn); + RexNode pulledFilterExpr = RexUtil.pullFactors(cluster.getRexBuilder(), convertedFilterExpr); RelNode filterRel = new HiveFilterRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), - srcRel, convertedFilterExpr); + srcRel, pulledFilterExpr); this.relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); relToHiveRR.put(filterRel, relToHiveRR.get(srcRel)); relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap);