diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index 58a7cff..3c3d634 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -623,7 +623,7 @@ public static boolean isComparisonOp(RexCall call) { return call.getKind().belongsTo(SqlKind.COMPARISON); } - private static final Function REX_STR_FN = new Function() { + public static final Function REX_STR_FN = new Function() { public String apply(RexNode r) { return r.toString(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java index f75303f..889d97f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java @@ -19,7 +19,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.RelOptUtil; @@ -35,6 +37,7 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; public class HiveRexUtil { @@ -145,9 +148,10 @@ private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call) { public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { final List terms = RelOptUtil.conjunctions(e); final List notTerms = new ArrayList<>(); + final List negatedTerms = new ArrayList<>(); final List nullOperands = new ArrayList<>(); final List notNullOperands = new ArrayList<>(); - final List comparedOperands = new ArrayList<>(); + final Set comparedOperands = new HashSet<>(); for (int i = 0; i < terms.size(); i++) { final RexNode term = terms.get(i); switch (term.getKind()) { @@ -186,6 +190,14 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { RexCall rightCast = (RexCall) right; comparedOperands.add(rightCast.getOperands().get(0)); } + RexCall negatedTerm = negate(rexBuilder, call); + if (negatedTerm != null) { + negatedTerms.add(negatedTerm); + RexCall invertNegatedTerm = invert(rexBuilder, negatedTerm); + if (invertNegatedTerm != null) { + negatedTerms.add(invertNegatedTerm); + } + } break; case IN: comparedOperands.add(((RexCall) term).operands.get(0)); @@ -230,9 +242,12 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { // Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable // Example #2. x AND y AND NOT (x AND y) - not satisfiable // Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable + final Set termsSet = new HashSet( + Lists.transform(terms, HiveCalciteUtil.REX_STR_FN)); for (RexNode notDisjunction : notTerms) { - final List terms2 = RelOptUtil.conjunctions(notDisjunction); - if (terms.containsAll(terms2)) { + final Set notSet = new HashSet( + Lists.transform(RelOptUtil.conjunctions(notDisjunction), HiveCalciteUtil.REX_STR_FN)); + if (termsSet.containsAll(notSet)) { return rexBuilder.makeLiteral(false); } } @@ -242,6 +257,14 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { rexBuilder.makeCall( SqlStdOperatorTable.NOT, notDisjunction)); } + // The negated terms + for (RexNode notDisjunction : negatedTerms) { + final Set notSet = new HashSet( + Lists.transform(RelOptUtil.conjunctions(notDisjunction), HiveCalciteUtil.REX_STR_FN)); + if (termsSet.containsAll(notSet)) { + return rexBuilder.makeLiteral(false); + } + } return RexUtil.composeConjunction(rexBuilder, terms, false); } @@ -263,7 +286,40 @@ public static RexNode simplifyOr(RexBuilder rexBuilder, RexCall call) { } return RexUtil.composeDisjunction(rexBuilder, terms, false); } - - - + + private static RexCall negate(RexBuilder rexBuilder, RexCall call) { + switch (call.getKind()) { + case EQUALS: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, call.getOperands()); + case NOT_EQUALS: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, call.getOperands()); + case LESS_THAN: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, call.getOperands()); + case GREATER_THAN: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, call.getOperands()); + case LESS_THAN_OR_EQUAL: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, call.getOperands()); + case GREATER_THAN_OR_EQUAL: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, call.getOperands()); + } + return null; + } + + private static RexCall invert(RexBuilder rexBuilder, RexCall call) { + switch (call.getKind()) { + case LESS_THAN: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, + Lists.reverse(call.getOperands())); + case GREATER_THAN: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, + Lists.reverse(call.getOperands())); + case LESS_THAN_OR_EQUAL: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + Lists.reverse(call.getOperands())); + case GREATER_THAN_OR_EQUAL: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + Lists.reverse(call.getOperands())); + } + return null; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java index 994af97..65a19e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java @@ -79,7 +79,7 @@ public HiveJoinPushTransitivePredicatesRule(Class clazz, @Override public void onMatch(RelOptRuleCall call) { Join join = call.rel(0); - + RelOptPredicateList preds = RelMetadataQuery.instance().getPulledUpPredicates(join); HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 1d2c4cc..9006f45 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -142,17 +142,11 @@ public FilterReduceExpressionsRule(Class filterClass, // predicate to see if it was already a constant, // in which case we don't need any runtime decision // about filtering. + // TODO: support LogicalValues if (newConditionExp.isAlwaysTrue()) { call.transformTo( filter.getInput()); - } - // TODO: support LogicalValues - else if (newConditionExp instanceof RexLiteral - || RexUtil.isNullLiteral(newConditionExp, true)) { - // call.transformTo(call.builder().values(filter.getRowType()).build()); - return; - } - else if (reduced + } else if (reduced || !newConditionExp.toString().equals(filter.getCondition().toString())) { call.transformTo(call.builder(). push(filter.getInput()).filter(newConditionExp).build()); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index 36d0b45..c5b3cca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -17,6 +17,15 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Predicate1; @@ -39,6 +48,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexPermutationShuttle; import org.apache.calcite.rex.RexPermuteInputsShuttle; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -61,18 +71,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; - //TODO: Move this to calcite public class HiveRelMdPredicates extends RelMdPredicates { + public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( BuiltInMethod.PREDICATES.method, new HiveRelMdPredicates()); @@ -333,7 +335,10 @@ public RelOptPredicateList inferPredicates( for (RexNode iP : inferredPredicates) { ImmutableBitSet iPBitSet = RelOptUtil.InputFinder.bits(iP); - if (leftFieldsBitSet.contains(iPBitSet)) { + if (iPBitSet.isEmpty()) { + leftInferredPredicates.add(iP); + rightInferredPredicates.add(iP); + } else if (leftFieldsBitSet.contains(iPBitSet)) { leftInferredPredicates.add(iP.accept(leftPermute)); } else if (rightFieldsBitSet.contains(iPBitSet)) { rightInferredPredicates.add(iP.accept(rightPermute)); @@ -382,6 +387,11 @@ private void infer(RexNode predicates, Set allExprsDigests, List inferedPredicates, boolean includeEqualityInference, ImmutableBitSet inferringFields) { for (RexNode r : RelOptUtil.conjunctions(predicates)) { + if (r instanceof RexLiteral) { + inferedPredicates.add(r); + allExprsDigests.add(r.toString()); + continue; + } if (!includeEqualityInference && equalityPredicates.contains(r.toString())) { continue; diff --git ql/src/test/queries/clientpositive/constprog3.q ql/src/test/queries/clientpositive/constprog3.q new file mode 100644 index 0000000..2911fe2 --- /dev/null +++ ql/src/test/queries/clientpositive/constprog3.q @@ -0,0 +1,8 @@ +create temporary table table1(id int, val int, val1 int, dimid int); +create temporary table table3(id int, val int, val1 int); + +explain +select table1.id, table1.val, table1.val1 +from table1 inner join table3 +on table1.dimid = table3.id and table3.id = 1 where table1.dimid <> 1; + diff --git ql/src/test/results/clientpositive/constprog3.q.out ql/src/test/results/clientpositive/constprog3.q.out new file mode 100644 index 0000000..e01a733 --- /dev/null +++ ql/src/test/results/clientpositive/constprog3.q.out @@ -0,0 +1,83 @@ +PREHOOK: query: create temporary table table1(id int, val int, val1 int, dimid int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table1 +POSTHOOK: query: create temporary table table1(id int, val int, val1 int, dimid int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table1 +PREHOOK: query: create temporary table table3(id int, val int, val1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table3 +POSTHOOK: query: create temporary table table3(id int, val int, val1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table3 +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: explain +select table1.id, table1.val, table1.val1 +from table1 inner join table3 +on table1.dimid = table3.id and table3.id = 1 where table1.dimid <> 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select table1.id, table1.val, table1.val1 +from table1 inner join table3 +on table1.dimid = table3.id and table3.id = 1 where table1.dimid <> 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: table1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: id (type: int), val (type: int), val1 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) + TableScan + alias: table3 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +