diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java new file mode 100644 index 0000000000..0ca7927527 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import org.apache.calcite.util.Util; + +/** + * Holder for a list of constants describing which bugs which have not been + * fixed. + * + *

The usage of the constant is a convenient way to identify the impact of + * the bug. When someone fixes the bug, they will remove the constant and all + * usages of it. Also, the constant helps track the propagation of the fix: as + * the fix is integrated into other branches, the constant will be removed from + * those branches.

+ * + */ +public final class Bug { + + /** + * Whether issue + * CALCITE-3982 is fixed. + */ + public static final boolean CALCITE_3982_FIXED = false; + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java new file mode 100644 index 0000000000..0c251a4f9e --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import java.util.List; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLocalRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexProgramBuilder; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; + +/** + * Mostly a copy of {@link org.apache.calcite.rel.rules.FilterMergeRule}. + * However, it flattens the predicate before creating the new filter. + */ +public class HiveFilterMergeRule extends RelOptRule { + + public static final HiveFilterMergeRule INSTANCE = + new HiveFilterMergeRule(); + + /** Private constructor. */ + private HiveFilterMergeRule() { + super(operand(HiveFilter.class, + operand(HiveFilter.class, any())), + HiveRelFactories.HIVE_BUILDER, null); + if (Bug.CALCITE_3982_FIXED) { + throw new AssertionError("Remove logic in HiveFilterMergeRule when [CALCITE-3982] " + + "has been fixed and use directly Calcite's FilterMergeRule instead."); + } + } + + //~ Methods ---------------------------------------------------------------- + + public void onMatch(RelOptRuleCall call) { + final HiveFilter topFilter = call.rel(0); + final HiveFilter bottomFilter = call.rel(1); + + RexBuilder rexBuilder = topFilter.getCluster().getRexBuilder(); + RexProgram bottomProgram = createProgram(bottomFilter); + RexProgram topProgram = createProgram(topFilter); + + RexProgram mergedProgram = + RexProgramBuilder.mergePrograms( + topProgram, + bottomProgram, + rexBuilder); + + RexNode newCondition = expandLocalRef(rexBuilder, + mergedProgram.getCondition(), mergedProgram.getExprList()); + + final RelBuilder relBuilder = call.builder(); + relBuilder.push(bottomFilter.getInput()) + .filter(newCondition); + + call.transformTo(relBuilder.build()); + } + + /** + * Creates a RexProgram corresponding to a LogicalFilter + * + * @param filterRel the LogicalFilter + * @return created RexProgram + */ + private RexProgram createProgram(Filter filterRel) { + RexProgramBuilder programBuilder = + new RexProgramBuilder( + filterRel.getRowType(), + filterRel.getCluster().getRexBuilder()); + programBuilder.addIdentity(); + programBuilder.addCondition(filterRel.getCondition()); + return programBuilder.getProgram(); + } + + private RexNode expandLocalRef(RexBuilder rexBuilder, + RexLocalRef ref, List exprs) { + return ref.accept(new ExpansionShuttle(rexBuilder, exprs)); + } + + private static class ExpansionShuttle extends RexShuttle { + private final RexBuilder rexBuilder; + private final List exprs; + + ExpansionShuttle(RexBuilder rexBuilder, List exprs) { + this.rexBuilder = rexBuilder; + this.exprs = exprs; + } + + @Override + public RexNode visitLocalRef(RexLocalRef localRef) { + RexNode tree = this.exprs.get(localRef.getIndex()); + return RexUtil.flatten(rexBuilder, tree.accept(this)); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index bf083067da..df9a21a400 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -207,6 +207,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFieldTrimmerRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterAggregateTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTSTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSetOpTransposeRule; @@ -2010,7 +2011,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv rules.add(HiveFilterJoinRule.FILTER_ON_JOIN); rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_BUILDER, Aggregate.class)); - rules.add(new FilterMergeRule(HiveRelFactories.HIVE_BUILDER)); + rules.add(HiveFilterMergeRule.INSTANCE); if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_REDUCE_WITH_STATS)) { rules.add(HiveReduceExpressionsWithStatsRule.INSTANCE); }