diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java index d50c517..f71d3f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -19,10 +19,8 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Set; -import org.apache.calcite.plan.Context; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; @@ -30,7 +28,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories.AggregateFactory; import org.apache.calcite.rel.core.RelFactories.FilterFactory; @@ -55,10 +52,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; -import org.apache.hadoop.hive.ql.optimizer.signature.RelTreeSignature; -import org.apache.hadoop.hive.ql.plan.mapper.StatsSource; -import org.apache.hadoop.hive.ql.stats.OperatorStats; - import com.google.common.collect.ImmutableList; public class HiveRelFactories { @@ -154,7 +147,6 @@ public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRelType joinType, Set variablesStopped, boolean semiJoinDone) { if (joinType == JoinRelType.SEMI) { - final JoinInfo joinInfo = JoinInfo.of(left, right, condition); final RelOptCluster cluster = left.getCluster(); return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); } @@ -167,7 +159,6 @@ // According to calcite, it is going to be removed before Calcite-2.0 // TODO: to handle CorrelationId if (joinType == JoinRelType.SEMI) { - final JoinInfo joinInfo = JoinInfo.of(left, right, condition); final RelOptCluster cluster = left.getCluster(); return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); } @@ -184,7 +175,6 @@ @Override public RelNode createSemiJoin(RelNode left, RelNode right, RexNode condition) { - final JoinInfo joinInfo = JoinInfo.of(left, right, condition); final RelOptCluster cluster = left.getCluster(); return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index d215736a..e647b88 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -55,7 +55,6 @@ import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexTableInputRef; import org.apache.calcite.rex.RexTableInputRef.RelTableRef; import org.apache.calcite.rex.RexUtil; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java index dfe2913..a1d617b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java @@ -505,7 +505,7 @@ * {@code e AND TRUE} becomes {@code e}; * {@code e AND e2 AND NOT e} becomes {@code e2}. */ public RexNode and(Iterable operands) { - return RexUtil.simplifyAnds(cluster.getRexBuilder(), operands); + return RexUtil.composeConjunction(cluster.getRexBuilder(), operands); } /** Creates an OR. */ @@ -756,13 +756,13 @@ * and optimized in a similar way to the {@link #and} method. * If the result is TRUE no filter is created. */ public HiveSubQRemoveRelBuilder filter(Iterable predicates) { - final RexNode x = RexUtil.simplifyAnds(cluster.getRexBuilder(), predicates, true); + final RexNode x = RexUtil.composeConjunction(cluster.getRexBuilder(), predicates); if (x.isAlwaysFalse()) { return empty(); } if (!x.isAlwaysTrue()) { final Frame frame = stack.pop(); - final RelNode filter = filterFactory.createFilter(frame.rel, x); + final RelNode filter = filterFactory.createFilter(frame.rel, x, ImmutableSet.of()); stack.push(new Frame(filter, frame.right)); } return this; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java index 8edd0b0..76ca54b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java @@ -45,7 +45,6 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlSplittableAggFunction; import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.Mappings; @@ -337,6 +336,8 @@ populateEquivalence(equivalence, ref1.getIndex(), ref0.getIndex()); } } + default: + break; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java index 7ea8154..d999ee4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java @@ -34,7 +34,9 @@ import org.apache.calcite.rex.RexExecutor; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexSimplify; +import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.Util; @@ -49,22 +51,22 @@ new HiveFilterSetOpTransposeRule(HiveRelFactories.HIVE_BUILDER); /** - * Creates a HiveFilterSetOpTransposeRule. - * This rule rewrites - * Fil - * | - * Union + * Creates a HiveFilterSetOpTransposeRule. + * This rule rewrites + * Fil + * | + * Union * / \ * Op1 Op2 - * - * to - * Union - * /\ - * FIL - * | | + * + * to + * Union + * /\ + * FIL + * | | * Op1 Op2 - * - * + * + * * It additionally can remove branch(es) of filter if its able to determine * that they are going to generate empty result set. */ @@ -115,9 +117,9 @@ listBuilder.add(newCondition); RexExecutor executor = Util.first(filterRel.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR); - final RexSimplify simplify = - new RexSimplify(rexBuilder, true, executor); - final RexNode x = simplify.simplifyAnds(listBuilder.build()); + final RexSimplify simplify = new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, executor); + final RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.AND, listBuilder.build()); + final RexNode x = simplify.simplifyUnknownAs(cond, RexUnknownAs.FALSE); if (x.isAlwaysFalse()) { // this is the last branch, and it is always false // We assume alwaysFalse filter will get pushed down to TS so this diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java index a657d13..05c7368 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java @@ -16,44 +16,26 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; import java.util.stream.Collectors; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelReferentialConstraint; -import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexPermuteInputsShuttle; -import org.apache.calcite.rex.RexTableInputRef; -import org.apache.calcite.rex.RexTableInputRef.RelTableRef; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Pair; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.MappingType; import org.apache.calcite.util.mapping.Mappings; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java index 48b7765..3569265 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java @@ -48,6 +48,7 @@ import org.apache.hive.common.util.AnnotationUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; /** @@ -104,15 +105,13 @@ if (!newLeftPredicate.isAlwaysTrue()) { RelNode curr = lChild; - lChild = filterFactory.createFilter( - lChild, newLeftPredicate.accept(new RexReplacer(lChild))); + lChild = filterFactory.createFilter(lChild, newLeftPredicate.accept(new RexReplacer(lChild)), ImmutableSet.of()); call.getPlanner().onCopy(curr, lChild); } if (!newRightPredicate.isAlwaysTrue()) { RelNode curr = rChild; - rChild = filterFactory.createFilter( - rChild, newRightPredicate.accept(new RexReplacer(rChild))); + rChild = filterFactory.createFilter(rChild, newRightPredicate.accept(new RexReplacer(rChild)), ImmutableSet.of()); call.getPlanner().onCopy(curr, rChild); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectOverIntersectRemoveRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectOverIntersectRemoveRule.java index 4d1347a..4a7a69f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectOverIntersectRemoveRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectOverIntersectRemoveRule.java @@ -19,16 +19,12 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rex.RexUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; -import com.google.common.base.Predicate; - /** * HiveProjectOverIntersectRemoveRule removes a HiveProject over another * HiveIntersect, provided the projects aren't projecting identical sets of diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java index cdc94d5..4078968 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Set; +import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; @@ -34,10 +35,11 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexSimplify; +import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.Pair; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -272,7 +274,11 @@ // If we did not reduce, check the children nodes RexNode node = super.visitCall(call); if (node != call) { - node = RexUtil.simplify(rexBuilder, node); + // FIXME if this rule will make some changes; then it will invoke simplify on all subtrees during exiting the recursion + RexSimplify simplify = + new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, filterOp.getCluster().getPlanner().getExecutor()); + node = simplify.simplifyUnknownAs(node,RexUnknownAs.UNKNOWN); + } return node; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index 068b687..4526fc6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -87,7 +87,6 @@ import org.apache.calcite.util.Bug; import org.apache.calcite.util.Holder; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; import org.apache.calcite.util.ReflectUtil; @@ -1907,7 +1906,7 @@ o1 = decorrFieldAccess((RexFieldAccess) o1); isCorrelated = true; } - if (isCorrelated && RexUtil.eq(o0, o1)) { + if (isCorrelated && o0.equals(o1)) { return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java index 2a2c384..04d688c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java @@ -577,7 +577,7 @@ } @Override public RexNode visitSubQuery(RexSubQuery subQuery) { - return RexUtil.eq(subQuery, this.subQuery) ? replacement : subQuery; + return subQuery.equals(this.subQuery) ? replacement : subQuery; } }