diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 4a7fc0d..eb0be3d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -91,7 +91,8 @@ public void initialize(HiveConf hiveConf) { transformations.add(new PartitionColumnsSeparator()); } - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD) && + !pctx.getContext().isCboSucceeded()) { transformations.add(new PredicateTransitivePropagate()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { transformations.add(new ConstantPropagate()); @@ -102,7 +103,7 @@ public void initialize(HiveConf hiveConf) { if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { // We run constant propagation twice because after predicate pushdown, filter expressions // are combined and may become eligible for reduction (like is not null filter). - transformations.add(new ConstantPropagate()); + transformations.add(new ConstantPropagate()); } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java new file mode 100644 index 0000000..a72385f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterMergeRule.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.rules.FilterMergeRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; + + +public class HiveFilterMergeRule extends FilterMergeRule { + + public static final HiveFilterMergeRule INSTANCE = new HiveFilterMergeRule( + HiveFilter.DEFAULT_FILTER_FACTORY); + + public HiveFilterMergeRule(RelFactories.FilterFactory filterFactory) { + super(filterFactory); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index 7e484b9..d5c8530 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -25,23 +25,37 @@ import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rex.RexNode; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule { + public static final HiveFilterProjectTransposeRule INSTANCE_DETERMINISTIC = + new HiveFilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, + HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY, true); + + public static final HiveFilterProjectTransposeRule INSTANCE = + new HiveFilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, + HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY, false); + + private final boolean onlyDeterministic; + public HiveFilterProjectTransposeRule(Class filterClass, FilterFactory filterFactory, Class projectClass, - ProjectFactory projectFactory) { + ProjectFactory projectFactory, boolean onlyDeterministic) { super(filterClass, filterFactory, projectClass, projectFactory); + this.onlyDeterministic = onlyDeterministic; } @Override public boolean matches(RelOptRuleCall call) { final Filter filterRel = call.rel(0); RexNode condition = filterRel.getCondition(); - if (!HiveCalciteUtil.isDeterministic(condition)) { + if (this.onlyDeterministic && !HiveCalciteUtil.isDeterministic(condition)) { return false; } return super.matches(call); } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index c005b1a..a2411e7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -63,7 +63,6 @@ import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.JoinToMultiJoinRule; import org.apache.calcite.rel.rules.LoptOptimizeJoinRule; import org.apache.calcite.rel.rules.ProjectMergeRule; @@ -138,6 +137,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateProjectMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExpandDistinctAggregatesRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSetOpTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveInsertExchange4JoinRule; @@ -484,6 +484,11 @@ static String canHandleQbForCbo(QueryProperties queryProperties, HiveConf conf, } @Override + boolean isCBOExecuted() { + return runCBO; + } + + @Override boolean continueJoinMerge() { return !(runCBO && disableSemJoinReordering); } @@ -1049,9 +1054,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE); // 3. Add not null filters - if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { - basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE); - } + basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE); // 4. Constant propagation, common filter extraction, and PPD basePlan = hepPlan(basePlan, true, mdProvider, @@ -1059,8 +1062,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv ReduceExpressionsRule.FILTER_INSTANCE, ReduceExpressionsRule.JOIN_INSTANCE, HivePreFilteringRule.INSTANCE, - new HiveFilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, - HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY), + HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC, new HiveFilterSetOpTransposeRule(HiveFilter.DEFAULT_FILTER_FACTORY), HiveFilterJoinRule.JOIN, HiveFilterJoinRule.FILTER_ON_JOIN, @@ -1083,8 +1085,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv // 7. Rerun PPD through Project as column pruning would have introduced DT // above scans basePlan = hepPlan(basePlan, true, mdProvider, - new FilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, - HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY)); + HiveFilterProjectTransposeRule.INSTANCE, HiveFilterMergeRule.INSTANCE); return basePlan; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0ff6001..38232d4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -55,7 +55,6 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ObjectPair; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StatsSetupConst.StatDB; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -7553,7 +7552,9 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, for (int i = 0; i < srcOps.length; i++) { // generate a ReduceSink operator for the join String[] srcs = baseSrc[i] != null ? new String[] {baseSrc[i]} : joinTree.getLeftAliases(); - srcOps[i] = genNotNullFilterForJoinSourcePlan(qb, srcOps[i], joinTree, joinKeys[i]); + if (!isCBOExecuted()) { + srcOps[i] = genNotNullFilterForJoinSourcePlan(qb, srcOps[i], joinTree, joinKeys[i]); + } srcOps[i] = genJoinReduceSinkChild(qb, joinKeys[i], srcOps[i], srcs, joinTree.getNextTag()); } @@ -8418,6 +8419,10 @@ private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[ return new ObjectPair(res, tgtToNodeExprMap); } + boolean isCBOExecuted() { + return false; + } + boolean continueJoinMerge() { return true; }