diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfigContext.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfigContext.java deleted file mode 100644 index 0e559e0..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfigContext.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite; - -import org.apache.calcite.plan.Context; -import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveAlgorithmsConf; - - -public class HiveConfigContext implements Context { - private HiveAlgorithmsConf config; - - public HiveConfigContext(HiveAlgorithmsConf config) { - this.config = config; - } - - public T unwrap(Class clazz) { - if (clazz.isInstance(config)) { - return clazz.cast(config); - } - return null; - } -} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveHepPlannerContext.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveHepPlannerContext.java new file mode 100644 index 0000000..ad79aee --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveHepPlannerContext.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import org.apache.calcite.plan.Context; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; + + +public class HiveHepPlannerContext implements Context { + private HiveRulesRegistry registry; + + public HiveHepPlannerContext(HiveRulesRegistry registry) { + this.registry = registry; + } + + public T unwrap(Class clazz) { + if (clazz.isInstance(registry)) { + return clazz.cast(registry); + } + return null; + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveVolcanoPlannerContext.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveVolcanoPlannerContext.java new file mode 100644 index 0000000..8859fc2 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveVolcanoPlannerContext.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import org.apache.calcite.plan.Context; +import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveAlgorithmsConf; + + +public class HiveVolcanoPlannerContext implements Context { + private HiveAlgorithmsConf config; + + public HiveVolcanoPlannerContext(HiveAlgorithmsConf config) { + this.config = config; + } + + public T unwrap(Class clazz) { + if (clazz.isInstance(config)) { + return clazz.cast(config); + } + return null; + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index a39ded2..8610edc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -22,7 +22,7 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.RelCollationTraitDef; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfigContext; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext; /** * Refinement of {@link org.apache.calcite.plan.volcano.VolcanoPlanner} for Hive. @@ -35,11 +35,11 @@ private static final boolean ENABLE_COLLATION_TRAIT = true; /** Creates a HiveVolcanoPlanner. */ - public HiveVolcanoPlanner(HiveConfigContext conf) { + public HiveVolcanoPlanner(HiveVolcanoPlannerContext conf) { super(HiveCost.FACTORY, conf); } - public static RelOptPlanner createPlanner(HiveConfigContext conf) { + public static RelOptPlanner createPlanner(HiveVolcanoPlannerContext conf) { final VolcanoPlanner planner = new HiveVolcanoPlanner(conf); planner.addRelTraitDef(ConventionTraitDef.INSTANCE); if (ENABLE_COLLATION_TRAIT) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java index 3e2311c..a9c539f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java @@ -31,7 +31,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.RelFactories.FilterFactory; -import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -76,14 +75,32 @@ private HivePreFilteringRule() { this.filterFactory = HiveFilter.DEFAULT_FILTER_FACTORY; } + @Override + public boolean matches(RelOptRuleCall call) { + final Filter filter = call.rel(0); + + HiveRulesRegistry registry = call.getPlanner(). + getContext().unwrap(HiveRulesRegistry.class); + + // If this operator has not been visited already by the rule, + // we need to apply the optimization + if (registry == null || !registry.getVisited(this).contains(filter)) { + return true; + } + + return false; + } + + @Override public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); final RelNode filterChild = call.rel(1); - // 0. If the filter is already on top of a TableScan, - // we can bail out - if (filterChild instanceof TableScan) { - return; + // 0. Register that we have visited this operator in this rule + HiveRulesRegistry registry = call.getPlanner(). + getContext().unwrap(HiveRulesRegistry.class); + if (registry != null) { + registry.registerVisited(this, filter); } final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRulesRegistry.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRulesRegistry.java new file mode 100644 index 0000000..18a065e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRulesRegistry.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import java.util.Set; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.rel.RelNode; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; + +public class HiveRulesRegistry { + + private SetMultimap registry; + + public HiveRulesRegistry() { + this.registry = HashMultimap.create(); + } + + public void registerVisited(RelOptRule rule, RelNode operator) { + this.registry.put(rule, operator); + } + + public Set getVisited(RelOptRule rule) { + return this.registry.get(rule); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 9c731b8..df7189a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -63,7 +63,6 @@ import org.apache.calcite.rel.metadata.CachingRelMetadataProvider; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; -import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.JoinToMultiJoinRule; @@ -118,8 +117,9 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfigContext; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; @@ -151,6 +151,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter; @@ -841,7 +842,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu final Double maxMemory = (double) HiveConf.getLongVar( conf, HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD); HiveAlgorithmsConf algorithmsConf = new HiveAlgorithmsConf(maxSplitSize, maxMemory); - HiveConfigContext confContext = new HiveConfigContext(algorithmsConf); + HiveVolcanoPlannerContext confContext = new HiveVolcanoPlannerContext(algorithmsConf); RelOptPlanner planner = HiveVolcanoPlanner.createPlanner(confContext); final RelOptQuery query = new RelOptQuery(planner); final RexBuilder rexBuilder = cluster.getRexBuilder(); @@ -1061,7 +1062,9 @@ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadata programBuilder.addRuleInstance(r); } - HepPlanner planner = new HepPlanner(programBuilder.build()); + HiveRulesRegistry registry = new HiveRulesRegistry(); + HiveHepPlannerContext context = new HiveHepPlannerContext(registry); + HepPlanner planner = new HepPlanner(programBuilder.build(), context); List list = Lists.newArrayList(); list.add(mdProvider); planner.registerMetadataProviders(list);