commit c461711117ac0f5514290a932697db90cf43878f Author: Ashutosh Chauhan Date: Wed Sep 3 14:24:44 2014 -0700 HIVE-7969 : Use Optiq's native FieldTrimmer instead of HiveRelFieldTrimmer diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index e9b258e..9faae39 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -6,7 +6,6 @@ import org.eigenbase.rel.RelFactories.ProjectFactory; import org.eigenbase.rel.RelNode; -import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeField; import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexInputRef; @@ -27,12 +26,12 @@ /** * Get list of virtual columns from the given list of projections. *

- * + * * @param exps * list of rex nodes representing projections * @return List of Virtual Columns, will not be null. */ - public static List getVirtualCols(List exps) { + public static List getVirtualCols(List exps) { List vCols = new ArrayList(); for (int i = 0; i < exps.size(); i++) { @@ -47,6 +46,7 @@ public static List getProjsFromBelowAsInputRef(final RelNode rel) { List projectList = Lists.transform(rel.getRowType().getFieldList(), new Function() { + @Override public RexNode apply(RelDataTypeField field) { return rel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); } @@ -73,7 +73,7 @@ public static void todo(String s) { /** * Push any equi join conditions that are not column references as Projections * on top of the children. - * + * * @param factory * Project factory to use. * @param inputRels diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java index e8069ee..d4bd678 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/TraitsUtil.java @@ -10,12 +10,10 @@ import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.reltype.RelDataType; -import org.eigenbase.rex.RexNode; public class TraitsUtil { - public static RelTraitSet getSelectTraitSet(RelOptCluster cluster, List exps, - RelNode child) { + public static RelTraitSet getSelectTraitSet(RelOptCluster cluster, RelNode child) { return cluster.traitSetOf(HiveRel.CONVENTION, RelCollationImpl.EMPTY); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java index 1588cdf..c81fd8a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveAggregateRel.java @@ -10,6 +10,7 @@ import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; import org.eigenbase.rel.InvalidRelException; +import org.eigenbase.rel.RelFactories.AggregateFactory; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.RelMetadataQuery; import org.eigenbase.relopt.RelOptCluster; @@ -19,12 +20,15 @@ public class HiveAggregateRel extends AggregateRelBase implements HiveRel { + public static final HiveAggRelFactory HIVE_AGGR_REL_FACTORY = new HiveAggRelFactory(); + public HiveAggregateRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, BitSet groupSet, List aggCalls) throws InvalidRelException { super(cluster, TraitsUtil.getAggregateTraitSet(cluster, traitSet, BitSets.toList(groupSet), aggCalls, child), child, groupSet, aggCalls); } + @Override public AggregateRelBase copy(RelTraitSet traitSet, RelNode input, BitSet groupSet, List aggCalls) { try { @@ -36,6 +40,7 @@ public AggregateRelBase copy(RelTraitSet traitSet, RelNode input, BitSet groupSe } } + @Override public void implement(Implementor implementor) { } @@ -49,4 +54,17 @@ public double getRows() { return RelMetadataQuery.getDistinctRowCount(this, groupSet, getCluster().getRexBuilder() .makeLiteral(true)); } + + private static class HiveAggRelFactory implements AggregateFactory { + + @Override + public RelNode createAggregate(RelNode child, BitSet groupSet, + List aggCalls) { + try { + return new HiveAggregateRel(child.getCluster(), child.getTraitSet(), child, groupSet, aggCalls); + } catch (InvalidRelException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java index 6a3410b..6f642b2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java @@ -1,9 +1,6 @@ package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators; -import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; -import java.util.List; import java.util.Set; import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; @@ -17,7 +14,6 @@ import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelOptCost; import org.eigenbase.relopt.RelOptPlanner; -import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeField; @@ -37,12 +33,12 @@ public enum MapJoinStreamingRelation { NONE, LEFT_RELATION, RIGHT_RELATION } - + public static final JoinFactory HIVE_JOIN_FACTORY = new HiveJoinFactoryImpl(); private final boolean m_leftSemiJoin; private final JoinAlgorithm m_joinAlgorithm; - private MapJoinStreamingRelation m_mapJoinStreamingSide = MapJoinStreamingRelation.NONE; + private final MapJoinStreamingRelation m_mapJoinStreamingSide = MapJoinStreamingRelation.NONE; public static HiveJoinRel getJoin(RelOptCluster cluster, RelNode left, RelNode right, RexNode condition, JoinRelType joinType, boolean leftSemiJoin) { @@ -99,11 +95,12 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { double leftRCount = RelMetadataQuery.getRowCount(getLeft()); double rightRCount = RelMetadataQuery.getRowCount(getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); - } + } /** * @return returns rowtype representing only the left join input */ + @Override public RelDataType deriveRowType() { if (m_leftSemiJoin) { return deriveJoinRowType(left.getRowType(), null, JoinRelType.INNER, @@ -116,7 +113,7 @@ public RelDataType deriveRowType() { private static class HiveJoinFactoryImpl implements JoinFactory { /** * Creates a join. - * + * * @param left * Left input * @param right diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java index 8cbf2f1..a60af2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveProjectRel.java @@ -34,7 +34,7 @@ /** * Creates a HiveProjectRel. - * + * * @param cluster * Cluster this relational expression belongs to * @param child @@ -47,14 +47,14 @@ * values as in {@link ProjectRelBase.Flags} */ public HiveProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, - List exps, RelDataType rowType, int flags) { + List exps, RelDataType rowType, int flags) { super(cluster, traitSet, child, exps, rowType, flags); m_virtualCols = ImmutableList.copyOf(HiveOptiqUtil.getVirtualCols(exps)); } /** * Creates a HiveProjectRel with no sort keys. - * + * * @param child * input relational expression * @param exps @@ -62,7 +62,7 @@ public HiveProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child * @param fieldNames * aliases of the expressions */ - public static HiveProjectRel create(RelNode child, List exps, List fieldNames) { + public static HiveProjectRel create(RelNode child, List exps, List fieldNames) { RelOptCluster cluster = child.getCluster(); RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), exps, fieldNames); return create(cluster, child, exps, rowType, Collections. emptyList()); @@ -71,9 +71,9 @@ public static HiveProjectRel create(RelNode child, List exps, List exps, + public static HiveProjectRel create(RelOptCluster cluster, RelNode child, List exps, RelDataType rowType, final List collationList) { - RelTraitSet traitSet = TraitsUtil.getSelectTraitSet(cluster, exps, child); + RelTraitSet traitSet = TraitsUtil.getSelectTraitSet(cluster, child); return new HiveProjectRel(cluster, traitSet, child, exps, rowType, Flags.BOXED); } @@ -125,9 +125,10 @@ public static RelNode projectMapping(RelNode rel, Mapping mapping, List outputProjList.add(rexBuilder.makeInputRef(rel, source)); } - return create(rel, (List) outputProjList, outputNameList); + return create(rel, outputProjList, outputNameList); } + @Override public ProjectRelBase copy(RelTraitSet traitSet, RelNode input, List exps, RelDataType rowType) { assert traitSet.containsIfApplicable(HiveRel.CONVENTION); @@ -139,6 +140,7 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { return HiveCost.FACTORY.makeZeroCost(); } + @Override public void implement(Implementor implementor) { } @@ -152,12 +154,14 @@ public void implement(Implementor implementor) { * . */ private static class HiveProjectFactoryImpl implements ProjectFactory { + @Override - public RelNode createProject(RelNode input, List exps, List fieldNames) { - RelNode project = HiveProjectRel.create(input, exps, fieldNames); + public RelNode createProject(RelNode child, + List childExprs, List fieldNames) { + RelNode project = HiveProjectRel.create(child, childExprs, fieldNames); // Make sure extra traits are carried over from the original rel - project = RelOptRule.convert(project, input.getTraitSet()); + project = RelOptRule.convert(project, child.getTraitSet()); return project; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java index 1c42a29..3bd7889 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java @@ -2,6 +2,7 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil; import org.eigenbase.rel.RelCollation; +import org.eigenbase.rel.RelFactories; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.SortRel; import org.eigenbase.relopt.RelOptCluster; @@ -9,12 +10,14 @@ import org.eigenbase.rex.RexNode; public class HiveSortRel extends SortRel implements HiveRel { - + + public static final HiveSortRelFactory HIVE_SORT_REL_FACTORY = new HiveSortRelFactory(); + public HiveSortRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RelCollation collation, RexNode offset, RexNode fetch) { super(cluster, TraitsUtil.getSortTraitSet(cluster, traitSet, collation), child, collation, offset, fetch); - + assert getConvention() == child.getConvention(); } @@ -31,6 +34,16 @@ public RexNode getFetchExpr() { return fetch; } + @Override public void implement(Implementor implementor) { } + + private static class HiveSortRelFactory implements RelFactories.SortFactory { + + @Override + public RelNode createSort(RelTraitSet traits, RelNode child, + RelCollation collation, RexNode offset, RexNode fetch) { + return new HiveSortRel(child.getCluster(), traits, child, collation, offset, fetch); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java index b81f3c8..ccd52b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveUnionRel.java @@ -3,14 +3,18 @@ import java.util.List; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel.Implementor; +import org.eigenbase.rel.RelFactories; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.SetOpRel; import org.eigenbase.rel.UnionRelBase; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; +import org.eigenbase.sql.SqlKind; public class HiveUnionRel extends UnionRelBase { + public static final HiveUnionRelFactory UNION_REL_FACTORY = new HiveUnionRelFactory(); + public HiveUnionRel(RelOptCluster cluster, RelTraitSet traits, List inputs) { super(cluster, traits, inputs, true); } @@ -22,4 +26,15 @@ public SetOpRel copy(RelTraitSet traitSet, List inputs, boolean all) { public void implement(Implementor implementor) { } + + private static class HiveUnionRelFactory implements RelFactories.SetOpFactory { + + @Override + public RelNode createSetOp(SqlKind kind, List inputs, boolean all) { + if (kind != SqlKind.UNION) { + throw new IllegalStateException("Expected to get Set operator of type Union. Found : " + kind); + } + return new HiveUnionRel(inputs.get(0).getCluster(), inputs.get(0).getTraitSet(), inputs); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java deleted file mode 100644 index c28f974..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java +++ /dev/null @@ -1,941 +0,0 @@ -package org.apache.hadoop.hive.ql.optimizer.optiq.rules; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -import net.hydromatic.linq4j.Ord; -import net.hydromatic.optiq.util.BitSets; - -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveAggregateRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveSortRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveUnionRel; -import org.eigenbase.rel.AggregateCall; -import org.eigenbase.rel.AggregateRel; -import org.eigenbase.rel.CalcRel; -import org.eigenbase.rel.InvalidRelException; -import org.eigenbase.rel.JoinRel; -import org.eigenbase.rel.JoinRelBase; -import org.eigenbase.rel.RelCollation; -import org.eigenbase.rel.RelFieldCollation; -import org.eigenbase.rel.RelNode; -import org.eigenbase.rel.SetOpRel; -import org.eigenbase.rel.SortRel; -import org.eigenbase.rel.TableFunctionRel; -import org.eigenbase.rel.TableModificationRel; -import org.eigenbase.rel.ValuesRel; -import org.eigenbase.rel.rules.RemoveTrivialProjectRule; -import org.eigenbase.rel.rules.SemiJoinRel; -import org.eigenbase.relopt.RelOptUtil; -import org.eigenbase.reltype.RelDataType; -import org.eigenbase.reltype.RelDataTypeField; -import org.eigenbase.reltype.RelDataTypeImpl; -import org.eigenbase.rex.RexBuilder; -import org.eigenbase.rex.RexLiteral; -import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexPermuteInputsShuttle; -import org.eigenbase.rex.RexUtil; -import org.eigenbase.rex.RexVisitor; -import org.eigenbase.sql.validate.SqlValidator; -import org.eigenbase.util.Bug; -import org.eigenbase.util.Pair; -import org.eigenbase.util.ReflectUtil; -import org.eigenbase.util.ReflectiveVisitor; -import org.eigenbase.util.Util; -import org.eigenbase.util.mapping.IntPair; -import org.eigenbase.util.mapping.Mapping; -import org.eigenbase.util.mapping.MappingType; -import org.eigenbase.util.mapping.Mappings; - -/** - * Transformer that walks over a tree of relational expressions, replacing each - * {@link RelNode} with a 'slimmed down' relational expression that projects - * only the columns required by its consumer. - * - *

- * Uses multi-methods to fire the right rule for each type of relational - * expression. This allows the transformer to be extended without having to add - * a new method to RelNode, and without requiring a collection of rule classes - * scattered to the four winds. - * - *

- * REVIEW: jhyde, 2009/7/28: Is sql2rel the correct package for this class? - * Trimming fields is not an essential part of SQL-to-Rel translation, and - * arguably belongs in the optimization phase. But this transformer does not - * obey the usual pattern for planner rules; it is difficult to do so, because - * each {@link RelNode} needs to return a different set of fields after - * trimming. - * - *

- * TODO: Change 2nd arg of the {@link #trimFields} method from BitSet to - * Mapping. Sometimes it helps the consumer if you return the columns in a - * particular order. For instance, it may avoid a project at the top of the tree - * just for reordering. Could ease the transition by writing methods that - * convert BitSet to Mapping and vice versa. - */ -public class HiveRelFieldTrimmer implements ReflectiveVisitor { - // ~ Static fields/initializers --------------------------------------------- - - // ~ Instance fields -------------------------------------------------------- - - private final ReflectUtil.MethodDispatcher trimFieldsDispatcher; - - // ~ Constructors ----------------------------------------------------------- - - /** - * Creates a RelFieldTrimmer. - * - * @param validator - * Validator - */ - public HiveRelFieldTrimmer(SqlValidator validator) { - Util.discard(validator); // may be useful one day - this.trimFieldsDispatcher = ReflectUtil.createMethodDispatcher( - TrimResult.class, this, "trimFields", RelNode.class, BitSet.class, - Set.class); - } - - // ~ Methods ---------------------------------------------------------------- - - /** - * Trims unused fields from a relational expression. - * - *

- * We presume that all fields of the relational expression are wanted by its - * consumer, so only trim fields that are not used within the tree. - * - * @param root - * Root node of relational expression - * @return Trimmed relational expression - */ - public RelNode trim(RelNode root) { - final int fieldCount = root.getRowType().getFieldCount(); - final BitSet fieldsUsed = BitSets.range(fieldCount); - final Set extraFields = Collections.emptySet(); - final TrimResult trimResult = dispatchTrimFields(root, fieldsUsed, - extraFields); - if (!trimResult.right.isIdentity()) { - throw new IllegalArgumentException(); - } - return trimResult.left; - } - - /** - * Trims the fields of an input relational expression. - * - * @param rel - * Relational expression - * @param input - * Input relational expression, whose fields to trim - * @param fieldsUsed - * Bitmap of fields needed by the consumer - * @return New relational expression and its field mapping - */ - protected TrimResult trimChild(RelNode rel, RelNode input, BitSet fieldsUsed, - Set extraFields) { - Util.discard(rel); - if (input.getClass().getName().endsWith("MedMdrClassExtentRel")) { - // MedMdrJoinRule cannot handle Join of Project of - // MedMdrClassExtentRel, only naked MedMdrClassExtentRel. - // So, disable trimming. - fieldsUsed = BitSets.range(input.getRowType().getFieldCount()); - } - return dispatchTrimFields(input, fieldsUsed, extraFields); - } - - /** - * Trims a child relational expression, then adds back a dummy project to - * restore the fields that were removed. - * - *

- * Sounds pointless? It causes unused fields to be removed further down the - * tree (towards the leaves), but it ensure that the consuming relational - * expression continues to see the same fields. - * - * @param rel - * Relational expression - * @param input - * Input relational expression, whose fields to trim - * @param fieldsUsed - * Bitmap of fields needed by the consumer - * @return New relational expression and its field mapping - */ - protected TrimResult trimChildRestore(RelNode rel, RelNode input, - BitSet fieldsUsed, Set extraFields) { - TrimResult trimResult = trimChild(rel, input, fieldsUsed, extraFields); - if (trimResult.right.isIdentity()) { - return trimResult; - } - final RelDataType rowType = input.getRowType(); - List fieldList = rowType.getFieldList(); - final List exprList = new ArrayList(); - final List nameList = rowType.getFieldNames(); - RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); - assert trimResult.right.getSourceCount() == fieldList.size(); - for (int i = 0; i < fieldList.size(); i++) { - int source = trimResult.right.getTargetOpt(i); - RelDataTypeField field = fieldList.get(i); - exprList.add(source < 0 ? rexBuilder.makeZeroLiteral(field.getType()) - : rexBuilder.makeInputRef(field.getType(), source)); - } - RelNode project = CalcRel - .createProject(trimResult.left, exprList, nameList); - return new TrimResult(project, Mappings.createIdentity(fieldList.size())); - } - - /** - * Invokes {@link #trimFields}, or the appropriate method for the type of the - * rel parameter, using multi-method dispatch. - * - * @param rel - * Relational expression - * @param fieldsUsed - * Bitmap of fields needed by the consumer - * @return New relational expression and its field mapping - */ - protected final TrimResult dispatchTrimFields(RelNode rel, BitSet fieldsUsed, - Set extraFields) { - final TrimResult trimResult = trimFieldsDispatcher.invoke(rel, fieldsUsed, - extraFields); - final RelNode newRel = trimResult.left; - final Mapping mapping = trimResult.right; - final int fieldCount = rel.getRowType().getFieldCount(); - assert mapping.getSourceCount() == fieldCount : "source: " - + mapping.getSourceCount() + " != " + fieldCount; - final int newFieldCount = newRel.getRowType().getFieldCount(); - assert mapping.getTargetCount() + extraFields.size() == newFieldCount : "target: " - + mapping.getTargetCount() - + " + " - + extraFields.size() - + " != " - + newFieldCount; - if (Bug.TODO_FIXED) { - assert newFieldCount > 0 : "rel has no fields after trim: " + rel; - } - if (newRel.equals(rel)) { - return new TrimResult(rel, mapping); - } - return trimResult; - } - - /** - * Visit method, per {@link org.eigenbase.util.ReflectiveVisitor}. - * - *

- * This method is invoked reflectively, so there may not be any apparent calls - * to it. The class (or derived classes) may contain overloads of this method - * with more specific types for the {@code rel} parameter. - * - *

- * Returns a pair: the relational expression created, and the mapping between - * the original fields and the fields of the newly created relational - * expression. - * - * @param rel - * Relational expression - * @param fieldsUsed - * Fields needed by the consumer - * @return relational expression and mapping - */ - public TrimResult trimFields(HiveRel rel, BitSet fieldsUsed, - Set extraFields) { - // We don't know how to trim this kind of relational expression, so give - // it back intact. - Util.discard(fieldsUsed); - return new TrimResult(rel, Mappings.createIdentity(rel.getRowType() - .getFieldCount())); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link HiveProjectRel} . - */ - public TrimResult trimFields(HiveProjectRel project, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = project.getRowType(); - final int fieldCount = rowType.getFieldCount(); - final RelNode input = project.getChild(); - final RelDataType inputRowType = input.getRowType(); - - // Which fields are required from the input? - BitSet inputFieldsUsed = new BitSet(inputRowType.getFieldCount()); - final Set inputExtraFields = new LinkedHashSet( - extraFields); - RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder( - inputFieldsUsed, inputExtraFields); - for (Ord ord : Ord.zip(project.getProjects())) { - if (fieldsUsed.get(ord.i)) { - ord.e.accept(inputFinder); - } - } - - // Create input with trimmed columns. - TrimResult trimResult = trimChild(project, input, inputFieldsUsed, - inputExtraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // If the input is unchanged, and we need to project all columns, - // there's nothing we can do. - if (newInput == input && fieldsUsed.cardinality() == fieldCount) { - return new TrimResult(project, Mappings.createIdentity(fieldCount)); - } - - // Some parts of the system can't handle rows with zero fields, so - // pretend that one field is used. - if (fieldsUsed.cardinality() == 0) { - final Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - fieldCount, 1); - final RexLiteral expr = project.getCluster().getRexBuilder() - .makeExactLiteral(BigDecimal.ZERO); - RelDataType newRowType = project - .getCluster() - .getTypeFactory() - .createStructType(Collections.singletonList(expr.getType()), - Collections.singletonList("DUMMY")); - HiveProjectRel newProject = new HiveProjectRel(project.getCluster(), - project.getCluster().traitSetOf(HiveRel.CONVENTION), newInput, - Collections. singletonList(expr), newRowType, - project.getFlags()); - return new TrimResult(newProject, mapping); - } - - // Build new project expressions, and populate the mapping. - List newProjectExprList = new ArrayList(); - final RexVisitor shuttle = new RexPermuteInputsShuttle( - inputMapping, newInput); - final Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - fieldCount, fieldsUsed.cardinality()); - for (Ord ord : Ord.zip(project.getProjects())) { - if (fieldsUsed.get(ord.i)) { - mapping.set(ord.i, newProjectExprList.size()); - RexNode newProjectExpr = ord.e.accept(shuttle); - newProjectExprList.add(newProjectExpr); - } - } - - final RelDataType newRowType = project.getCluster().getTypeFactory() - .createStructType(Mappings.apply3(mapping, rowType.getFieldList())); - - final List newCollations = RexUtil.apply(inputMapping, - project.getCollationList()); - - final RelNode newProject; - if (RemoveTrivialProjectRule.isIdentity(newProjectExprList, newRowType, - newInput.getRowType())) { - // The new project would be the identity. It is equivalent to return - // its child. - newProject = newInput; - } else { - newProject = new HiveProjectRel(project.getCluster(), project - .getCluster() - .traitSetOf( - newCollations.isEmpty() ? HiveRel.CONVENTION : newCollations - .get(0)), newInput, newProjectExprList, newRowType, - project.getFlags()); - assert newProject.getClass() == project.getClass(); - } - return new TrimResult(newProject, mapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link HiveFilterRel}. - */ - public TrimResult trimFields(HiveFilterRel filter, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = filter.getRowType(); - final int fieldCount = rowType.getFieldCount(); - final RexNode conditionExpr = filter.getCondition(); - final RelNode input = filter.getChild(); - - // We use the fields used by the consumer, plus any fields used in the - // filter. - BitSet inputFieldsUsed = (BitSet) fieldsUsed.clone(); - final Set inputExtraFields = new LinkedHashSet( - extraFields); - RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder( - inputFieldsUsed, inputExtraFields); - conditionExpr.accept(inputFinder); - - // Create input with trimmed columns. - TrimResult trimResult = trimChild(filter, input, inputFieldsUsed, - inputExtraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // If the input is unchanged, and we need to project all columns, - // there's nothing we can do. - if (newInput == input && fieldsUsed.cardinality() == fieldCount) { - return new TrimResult(filter, Mappings.createIdentity(fieldCount)); - } - - // Build new project expressions, and populate the mapping. - final RexVisitor shuttle = new RexPermuteInputsShuttle( - inputMapping, newInput); - RexNode newConditionExpr = conditionExpr.accept(shuttle); - - final HiveFilterRel newFilter = new HiveFilterRel(filter.getCluster(), - filter.getCluster().traitSetOf(HiveRel.CONVENTION), newInput, - newConditionExpr); - assert newFilter.getClass() == filter.getClass(); - - // The result has the same mapping as the input gave us. Sometimes we - // return fields that the consumer didn't ask for, because the filter - // needs them for its condition. - return new TrimResult(newFilter, inputMapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for {@link SortRel}. - */ - public TrimResult trimFields(HiveSortRel sort, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = sort.getRowType(); - final int fieldCount = rowType.getFieldCount(); - final RelCollation collation = sort.getCollation(); - final RelNode input = sort.getChild(); - - // We use the fields used by the consumer, plus any fields used as sort - // keys. - BitSet inputFieldsUsed = (BitSet) fieldsUsed.clone(); - for (RelFieldCollation field : collation.getFieldCollations()) { - inputFieldsUsed.set(field.getFieldIndex()); - } - - // Create input with trimmed columns. - final Set inputExtraFields = Collections.emptySet(); - TrimResult trimResult = trimChild(sort, input, inputFieldsUsed, - inputExtraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // If the input is unchanged, and we need to project all columns, - // there's nothing we can do. - if (newInput == input && inputMapping.isIdentity() - && fieldsUsed.cardinality() == fieldCount) { - return new TrimResult(sort, Mappings.createIdentity(fieldCount)); - } - - final SortRel newSort = sort.copy(sort.getTraitSet(), newInput, - RexUtil.apply(inputMapping, collation)); - assert newSort.getClass() == sort.getClass(); - - // The result has the same mapping as the input gave us. Sometimes we - // return fields that the consumer didn't ask for, because the filter - // needs them for its condition. - return new TrimResult(newSort, inputMapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for {@link JoinRel}. - * - * Have to do this because of the way ReflectUtil works. - if there is an - * exact match, things are fine. - otherwise it doesn't allow any ambiguity(in - * this case between a superClass(JoinRelBase) and an interface(HiveRel). - */ - private TrimResult _trimFields(JoinRelBase join, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = join.getRowType(); - final int fieldCount = join.getSystemFieldList().size() + - join.getLeft().getRowType().getFieldCount() + - join.getRight().getRowType().getFieldCount(); - final RexNode conditionExpr = join.getCondition(); - final int systemFieldCount = join.getSystemFieldList().size(); - - // Add in fields used in the condition. - BitSet fieldsUsedPlus = (BitSet) fieldsUsed.clone(); - final Set combinedInputExtraFields = new LinkedHashSet( - extraFields); - RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder( - fieldsUsedPlus, combinedInputExtraFields); - conditionExpr.accept(inputFinder); - - // If no system fields are used, we can remove them. - int systemFieldUsedCount = 0; - for (int i = 0; i < systemFieldCount; ++i) { - if (fieldsUsed.get(i)) { - ++systemFieldUsedCount; - } - } - final int newSystemFieldCount; - if (systemFieldUsedCount == 0) { - newSystemFieldCount = 0; - } else { - newSystemFieldCount = systemFieldCount; - } - - int offset = systemFieldCount; - int changeCount = 0; - int newFieldCount = newSystemFieldCount; - List newInputs = new ArrayList(2); - List inputMappings = new ArrayList(); - List inputExtraFieldCounts = new ArrayList(); - for (RelNode input : join.getInputs()) { - final RelDataType inputRowType = input.getRowType(); - final int inputFieldCount = inputRowType.getFieldCount(); - - // Compute required mapping. - BitSet inputFieldsUsed = new BitSet(inputFieldCount); - for (int bit : BitSets.toIter(fieldsUsedPlus)) { - if (bit >= offset && bit < offset + inputFieldCount) { - inputFieldsUsed.set(bit - offset); - } - } - - // If there are system fields, we automatically use the - // corresponding field in each input. - if (newSystemFieldCount > 0) { - // calling with newSystemFieldCount == 0 should be safe but hits - // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6222207 - inputFieldsUsed.set(0, newSystemFieldCount); - } - - // FIXME: We ought to collect extra fields for each input - // individually. For now, we assume that just one input has - // on-demand fields. - Set inputExtraFields = RelDataTypeImpl.extra(rowType) == null ? Collections - . emptySet() : combinedInputExtraFields; - inputExtraFieldCounts.add(inputExtraFields.size()); - - // Cross Join may not reference any thing from below - // Ex: select R1.x from r1 join r2; - if (inputExtraFields.size() == 0 && inputFieldsUsed.isEmpty()) { - inputFieldsUsed.set(0); - } - - TrimResult trimResult = trimChild(join, input, inputFieldsUsed, - inputExtraFields); - newInputs.add(trimResult.left); - if (trimResult.left != input) { - ++changeCount; - } - - final Mapping inputMapping = trimResult.right; - inputMappings.add(inputMapping); - - // Move offset to point to start of next input. - offset += inputFieldCount; - newFieldCount += inputMapping.getTargetCount() + inputExtraFields.size(); - } - - Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - fieldCount, newFieldCount); - for (int i = 0; i < newSystemFieldCount; ++i) { - mapping.set(i, i); - } - offset = systemFieldCount; - int newOffset = newSystemFieldCount; - for (int i = 0; i < inputMappings.size(); i++) { - Mapping inputMapping = inputMappings.get(i); - for (IntPair pair : inputMapping) { - mapping.set(pair.source + offset, pair.target + newOffset); - } - offset += inputMapping.getSourceCount(); - newOffset += inputMapping.getTargetCount() + inputExtraFieldCounts.get(i); - } - - if (changeCount == 0 && mapping.isIdentity()) { - return new TrimResult(join, Mappings.createIdentity(fieldCount)); - } - - // Build new join. - final RexVisitor shuttle = new RexPermuteInputsShuttle(mapping, - newInputs.get(0), newInputs.get(1)); - RexNode newConditionExpr = conditionExpr.accept(shuttle); - - final JoinRelBase newJoin = join.copy(join.getTraitSet(), newConditionExpr, - newInputs.get(0), newInputs.get(1), join.getJoinType(), false); - - /* - * For SemiJoins only map fields from the left-side - */ - if ( join instanceof SemiJoinRel ) { - Mapping inputMapping = inputMappings.get(0); - mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - join.getRowType().getFieldCount(), newSystemFieldCount + inputMapping.getTargetCount()); - for (int i = 0; i < newSystemFieldCount; ++i) { - mapping.set(i, i); - } - offset = systemFieldCount; - newOffset = newSystemFieldCount; - for (IntPair pair : inputMapping) { - mapping.set(pair.source + offset, pair.target + newOffset); - } - } - - return new TrimResult(newJoin, mapping); - } - - public TrimResult trimFields(HiveJoinRel join, BitSet fieldsUsed, - Set extraFields) { - return _trimFields(join, fieldsUsed, extraFields); - } - - public TrimResult trimFields(SemiJoinRel join, BitSet fieldsUsed, - Set extraFields) { - return _trimFields(join, fieldsUsed, extraFields); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for {@link SetOpRel} - * (including UNION and UNION ALL). - */ - public TrimResult trimFields(HiveUnionRel setOp, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = setOp.getRowType(); - final int fieldCount = rowType.getFieldCount(); - int changeCount = 0; - - // Fennel abhors an empty row type, so pretend that the parent rel - // wants the last field. (The last field is the least likely to be a - // system field.) - if (fieldsUsed.isEmpty()) { - fieldsUsed.set(rowType.getFieldCount() - 1); - } - - // Compute the desired field mapping. Give the consumer the fields they - // want, in the order that they appear in the bitset. - final Mapping mapping = createMapping(fieldsUsed, fieldCount); - - // Create input with trimmed columns. - final List newInputs = new ArrayList(); - for (RelNode input : setOp.getInputs()) { - TrimResult trimResult = trimChild(setOp, input, fieldsUsed, extraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // We want "mapping", the input gave us "inputMapping", compute - // "remaining" mapping. - // | | | - // |---------------- mapping ---------->| - // |-- inputMapping -->| | - // | |-- remaining -->| - // - // For instance, suppose we have columns [a, b, c, d], - // the consumer asked for mapping = [b, d], - // and the transformed input has columns inputMapping = [d, a, b]. - // remaining will permute [b, d] to [d, a, b]. - Mapping remaining = Mappings.divide(mapping, inputMapping); - - // Create a projection; does nothing if remaining is identity. - newInput = HiveProjectRel.projectMapping(newInput, remaining, null); - - if (input != newInput) { - ++changeCount; - } - newInputs.add(newInput); - } - - // If the input is unchanged, and we need to project all columns, - // there's to do. - if (changeCount == 0 && mapping.isIdentity()) { - return new TrimResult(setOp, mapping); - } - - RelNode newSetOp = setOp.copy(setOp.getTraitSet(), newInputs, true); - return new TrimResult(newSetOp, mapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link AggregateRel}. - * - * @throws InvalidRelException - */ - public TrimResult trimFields(HiveAggregateRel aggregate, BitSet fieldsUsed, - Set extraFields) throws InvalidRelException { - // Fields: - // - // | sys fields | group fields | agg functions | - // - // Two kinds of trimming: - // - // 1. If agg rel has system fields but none of these are used, create an - // agg rel with no system fields. - // - // 2. If aggregate functions are not used, remove them. - // - // But grouping fields stay, even if they are not used. - - final RelDataType rowType = aggregate.getRowType(); - - // Compute which input fields are used. - BitSet inputFieldsUsed = new BitSet(); - // 1. group fields are always used - for (int i : BitSets.toIter(aggregate.getGroupSet())) { - inputFieldsUsed.set(i); - } - // 2. agg functions - for (AggregateCall aggCall : aggregate.getAggCallList()) { - for (int i : aggCall.getArgList()) { - inputFieldsUsed.set(i); - } - } - - // Create input with trimmed columns. - final RelNode input = aggregate.getInput(0); - final Set inputExtraFields = Collections.emptySet(); - final TrimResult trimResult = trimChild(aggregate, input, inputFieldsUsed, - inputExtraFields); - final RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // If the input is unchanged, and we need to project all columns, - // there's nothing to do. - if (input == newInput - && fieldsUsed.equals(BitSets.range(rowType.getFieldCount()))) { - return new TrimResult(aggregate, Mappings.createIdentity(rowType - .getFieldCount())); - } - - // Which agg calls are used by our consumer? - final int groupCount = aggregate.getGroupSet().cardinality(); - int j = groupCount; - int usedAggCallCount = 0; - for (int i = 0; i < aggregate.getAggCallList().size(); i++) { - if (fieldsUsed.get(j++)) { - ++usedAggCallCount; - } - } - - // Offset due to the number of system fields having changed. - Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - rowType.getFieldCount(), groupCount + usedAggCallCount); - - final BitSet newGroupSet = Mappings.apply(inputMapping, - aggregate.getGroupSet()); - - // Populate mapping of where to find the fields. System and grouping - // fields first. - for (IntPair pair : inputMapping) { - if (pair.source < groupCount) { - mapping.set(pair.source, pair.target); - } - } - - // Now create new agg calls, and populate mapping for them. - final List newAggCallList = new ArrayList(); - j = groupCount; - for (AggregateCall aggCall : aggregate.getAggCallList()) { - if (fieldsUsed.get(j)) { - AggregateCall newAggCall = new AggregateCall(aggCall.getAggregation(), - aggCall.isDistinct(), Mappings.apply2(inputMapping, - aggCall.getArgList()), aggCall.getType(), aggCall.getName()); - if (newAggCall.equals(aggCall)) { - newAggCall = aggCall; // immutable -> canonize to save space - } - mapping.set(j, groupCount + newAggCallList.size()); - newAggCallList.add(newAggCall); - } - ++j; - } - - RelNode newAggregate = new HiveAggregateRel(aggregate.getCluster(), - aggregate.getCluster().traitSetOf(HiveRel.CONVENTION), newInput, - newGroupSet, newAggCallList); - - assert newAggregate.getClass() == aggregate.getClass(); - - return new TrimResult(newAggregate, mapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link TableModificationRel}. - */ - public TrimResult trimFields(TableModificationRel modifier, - BitSet fieldsUsed, Set extraFields) { - // Ignore what consumer wants. We always project all columns. - Util.discard(fieldsUsed); - - final RelDataType rowType = modifier.getRowType(); - final int fieldCount = rowType.getFieldCount(); - RelNode input = modifier.getChild(); - - // We want all fields from the child. - final int inputFieldCount = input.getRowType().getFieldCount(); - BitSet inputFieldsUsed = BitSets.range(inputFieldCount); - - // Create input with trimmed columns. - final Set inputExtraFields = Collections.emptySet(); - TrimResult trimResult = trimChild(modifier, input, inputFieldsUsed, - inputExtraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - if (!inputMapping.isIdentity()) { - // We asked for all fields. Can't believe that the child decided - // to permute them! - throw Util.newInternal("Expected identity mapping, got " + inputMapping); - } - - TableModificationRel newModifier = modifier; - if (newInput != input) { - newModifier = modifier.copy(modifier.getTraitSet(), - Collections.singletonList(newInput)); - } - assert newModifier.getClass() == modifier.getClass(); - - // Always project all fields. - Mapping mapping = Mappings.createIdentity(fieldCount); - return new TrimResult(newModifier, mapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link TableFunctionRel}. - */ - public TrimResult trimFields(TableFunctionRel tabFun, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = tabFun.getRowType(); - final int fieldCount = rowType.getFieldCount(); - List newInputs = new ArrayList(); - - for (RelNode input : tabFun.getInputs()) { - final int inputFieldCount = input.getRowType().getFieldCount(); - BitSet inputFieldsUsed = BitSets.range(inputFieldCount); - - // Create input with trimmed columns. - final Set inputExtraFields = Collections.emptySet(); - TrimResult trimResult = trimChildRestore(tabFun, input, inputFieldsUsed, - inputExtraFields); - assert trimResult.right.isIdentity(); - newInputs.add(trimResult.left); - } - - TableFunctionRel newTabFun = tabFun; - if (!tabFun.getInputs().equals(newInputs)) { - newTabFun = tabFun.copy(tabFun.getTraitSet(), newInputs); - } - assert newTabFun.getClass() == tabFun.getClass(); - - // Always project all fields. - Mapping mapping = Mappings.createIdentity(fieldCount); - return new TrimResult(newTabFun, mapping); - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link org.eigenbase.rel.ValuesRel}. - */ - public TrimResult trimFields(ValuesRel values, BitSet fieldsUsed, - Set extraFields) { - final RelDataType rowType = values.getRowType(); - final int fieldCount = rowType.getFieldCount(); - - // If they are asking for no fields, we can't give them what they want, - // because zero-column records are illegal. Give them the last field, - // which is unlikely to be a system field. - if (fieldsUsed.isEmpty()) { - fieldsUsed = BitSets.range(fieldCount - 1, fieldCount); - } - - // If all fields are used, return unchanged. - if (fieldsUsed.equals(BitSets.range(fieldCount))) { - Mapping mapping = Mappings.createIdentity(fieldCount); - return new TrimResult(values, mapping); - } - - List> newTuples = new ArrayList>(); - for (List tuple : values.getTuples()) { - List newTuple = new ArrayList(); - for (int field : BitSets.toIter(fieldsUsed)) { - newTuple.add(tuple.get(field)); - } - newTuples.add(newTuple); - } - - final Mapping mapping = createMapping(fieldsUsed, fieldCount); - RelDataType newRowType = values.getCluster().getTypeFactory() - .createStructType(Mappings.apply3(mapping, rowType.getFieldList())); - final ValuesRel newValues = new ValuesRel(values.getCluster(), newRowType, - newTuples); - return new TrimResult(newValues, mapping); - } - - private Mapping createMapping(BitSet fieldsUsed, int fieldCount) { - final Mapping mapping = Mappings.create(MappingType.INVERSE_SURJECTION, - fieldCount, fieldsUsed.cardinality()); - int i = 0; - for (int field : BitSets.toIter(fieldsUsed)) { - mapping.set(field, i++); - } - return mapping; - } - - /** - * Variant of {@link #trimFields(RelNode, BitSet, Set)} for - * {@link org.eigenbase.rel.TableAccessRel}. - */ - public TrimResult trimFields(final HiveTableScanRel tableAccessRel, - BitSet fieldsUsed, Set extraFields) { - final int fieldCount = tableAccessRel.getRowType().getFieldCount(); - if (fieldsUsed.isEmpty() || (fieldsUsed.equals(BitSets.range(fieldCount)) && extraFields.isEmpty())) { - return trimFields((HiveRel) tableAccessRel, fieldsUsed, extraFields); - } - final RelNode _newTableAccessRel = tableAccessRel.project(fieldsUsed, - extraFields); - final RelNode newTableAccessRel = HiveProjectRel.DEFAULT_PROJECT_FACTORY - .createProject(tableAccessRel, _newTableAccessRel.getChildExps(), - _newTableAccessRel.getRowType().getFieldNames()); - final Mapping mapping = createMapping(fieldsUsed, fieldCount); - return new TrimResult(newTableAccessRel, mapping); - } - - // ~ Inner Classes ---------------------------------------------------------- - - /** - * Result of an attempt to trim columns from a relational expression. - * - *

- * The mapping describes where to find the columns wanted by the parent of the - * current relational expression. - * - *

- * The mapping is a {@link org.eigenbase.util.mapping.Mappings.SourceMapping}, - * which means that no column can be used more than once, and some columns are - * not used. {@code columnsUsed.getSource(i)} returns the source of the i'th - * output field. - * - *

- * For example, consider the mapping for a relational expression that has 4 - * output columns but only two are being used. The mapping {2 → 1, 3 - * → 0} would give the following behavior: - *

- * - * - */ - protected static class TrimResult extends Pair { - /** - * Creates a TrimResult. - * - * @param left - * New relational expression - * @param right - * Mapping of fields onto original fields - */ - public TrimResult(RelNode left, Mapping right) { - super(left, right); - } - } -} - -// End RelFieldTrimmer.java - diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8fe8b3c..335611d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -127,7 +127,6 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveUnionRel; import org.apache.hadoop.hive.ql.optimizer.optiq.rules.HivePartitionPrunerRule; import org.apache.hadoop.hive.ql.optimizer.optiq.rules.HivePushFilterPastJoinRule; -import org.apache.hadoop.hive.ql.optimizer.optiq.rules.HiveRelFieldTrimmer; import org.apache.hadoop.hive.ql.optimizer.optiq.translator.ASTConverter; import org.apache.hadoop.hive.ql.optimizer.optiq.translator.RexNodeConverter; import org.apache.hadoop.hive.ql.optimizer.optiq.translator.SqlFunctionConverter; @@ -231,6 +230,7 @@ import org.eigenbase.rel.JoinRelType; import org.eigenbase.rel.RelCollation; import org.eigenbase.rel.RelCollationImpl; +import org.eigenbase.rel.RelFactories; import org.eigenbase.rel.RelFieldCollation; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.CachingRelMetadataProvider; @@ -267,6 +267,7 @@ import org.eigenbase.sql.SqlWindow; import org.eigenbase.sql.parser.SqlParserPos; import org.eigenbase.sql.type.SqlTypeName; +import org.eigenbase.sql2rel.RelFieldTrimmer; import org.eigenbase.sql.SqlCall; import org.eigenbase.sql.SqlExplainLevel; import org.eigenbase.sql.SqlKind; @@ -11945,7 +11946,9 @@ public RelNode applyPreCBOTransforms(RelNode basePlan, RelMetadataProvider mdPro RemoveTrivialProjectRule.INSTANCE, new HivePartitionPrunerRule(SemanticAnalyzer.this.conf)); - HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null); + RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProjectRel.DEFAULT_PROJECT_FACTORY, + HiveFilterRel.DEFAULT_FILTER_FACTORY, HiveJoinRel.HIVE_JOIN_FACTORY, RelFactories.DEFAULT_SEMI_JOIN_FACTORY, + HiveSortRel.HIVE_SORT_REL_FACTORY, HiveAggregateRel.HIVE_AGGR_REL_FACTORY, HiveUnionRel.UNION_REL_FACTORY); basePlan = fieldTrimmer.trim(basePlan); return basePlan; @@ -12210,7 +12213,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ /** * Generate Join Logical Plan Relnode by walking through the join AST. - * + * * @param qb * @param aliasToRel * Alias(Table/Relation alias) to RelNode; only read and not @@ -12394,7 +12397,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, Map aliasToRel, boolean forHavingClause) throws SemanticException { /* * Handle Subquery predicates. - * + * * Notes (8/22/14 hb): Why is this a copy of the code from {@link * #genFilterPlan} - for now we will support the same behavior as non CBO * route. - but plan to allow nested SubQueries(Restriction.9.m) and @@ -12778,7 +12781,7 @@ private AggInfo getHiveAggInfo(ASTNode aggAst, int aggFnLstArgIndx, RowResolver /** * Generate GB plan. - * + * * @param qb * @param srcRel * @return TODO: 1. Grouping Sets (roll up..) @@ -13118,7 +13121,7 @@ int getWindowSpecIndx(ASTNode wndAST) { WindowFunctionSpec wFnSpec = (WindowFunctionSpec) wExpSpec; ASTNode windowProjAst = wFnSpec.getExpression(); // TODO: do we need to get to child? - int wndSpecASTIndx = getWindowSpecIndx((ASTNode) windowProjAst); + int wndSpecASTIndx = getWindowSpecIndx(windowProjAst); // 2. Get Hive Aggregate Info AggInfo hiveAggInfo = getHiveAggInfo(windowProjAst, wndSpecASTIndx - 1, this.m_relToHiveRR.get(srcRel)); @@ -13251,7 +13254,7 @@ public String apply(String hName) { /** * NOTE: there can only be one select caluse since we don't handle multi * destination insert. - * + * * @throws SemanticException */ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticException {