diff --git a/pom.xml b/pom.xml index 848432c..15711cc 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ 3.4 1.7.7 0.8.0.RELEASE - 1.5.0 + 1.6.0-SNAPSHOT 4.2.1 4.1.6 4.1.7 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index 4825a61..8fe44ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -48,6 +48,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexRangeRef; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; @@ -629,7 +630,7 @@ public String apply(RexNode r) { }; public static ImmutableList getPredsNotPushedAlready(RelNode inp, List predsToPushDown) { - final RelOptPredicateList predicates = RelMetadataQuery.getPulledUpPredicates(inp); + final RelOptPredicateList predicates = RelMetadataQuery.instance().getPulledUpPredicates(inp); final ImmutableSet alreadyPushedPreds = ImmutableSet.copyOf(Lists.transform( predicates.pulledUpPredicates, REX_STR_FN)); final ImmutableList.Builder newConjuncts = ImmutableList.builder(); @@ -896,6 +897,12 @@ public Boolean visitFieldAccess(RexFieldAccess fieldAccess) { // ".FIELD" is constant iff "" is constant. return fieldAccess.getReferenceExpr().accept(this); } + + @Override + public Boolean visitSubQuery(RexSubQuery subQuery) { + // it seems that it is not used by anything. + return null; + } } public static Set getInputRefs(RexNode expr) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java index eeec44e..a4c6319 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories.AggregateFactory; @@ -147,6 +148,13 @@ public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRe Set variablesStopped, boolean semiJoinDone) { return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType, false); } + + @Override + public RelNode createJoin(RelNode left, RelNode right, RexNode condition, + Set variablesSet, JoinRelType joinType, boolean semiJoinDone) { + // According to calcite, it is going to be removed before Calcite-2.0 + return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType, semiJoinDone); + } } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java index 6840418..8c00322 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java @@ -200,7 +200,7 @@ public double computeSMBMapJoinIOCost( } public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { - Double currentMemory = RelMetadataQuery.cumulativeMemoryWithinPhase(input); + Double currentMemory = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; @@ -314,8 +314,8 @@ public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation strea if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.RIGHT_RELATION) { // Left side - final Double leftAvgRowSize = RelMetadataQuery.getAverageRowSize(join.getLeft()); - final Double leftRowCount = RelMetadataQuery.getRowCount(join.getLeft()); + final Double leftAvgRowSize = RelMetadataQuery.instance().getAverageRowSize(join.getLeft()); + final Double leftRowCount = RelMetadataQuery.instance().getRowCount(join.getLeft()); if (leftAvgRowSize == null || leftRowCount == null) { return null; } @@ -324,8 +324,8 @@ public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation strea if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.LEFT_RELATION) { // Right side - final Double rightAvgRowSize = RelMetadataQuery.getAverageRowSize(join.getRight()); - final Double rightRowCount = RelMetadataQuery.getRowCount(join.getRight()); + final Double rightAvgRowSize = RelMetadataQuery.instance().getAverageRowSize(join.getRight()); + final Double rightRowCount = RelMetadataQuery.instance().getRowCount(join.getRight()); if (rightAvgRowSize == null || rightRowCount == null) { return null; } @@ -338,8 +338,8 @@ public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits - final Double averageRowSize = RelMetadataQuery.getAverageRowSize(join); - final Double rowCount = RelMetadataQuery.getRowCount(join); + final Double averageRowSize = RelMetadataQuery.instance().getAverageRowSize(join); + final Double rowCount = RelMetadataQuery.instance().getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } @@ -357,7 +357,7 @@ public static Integer getSplitCountWithoutRepartition(HiveJoin join) { } else { return null; } - return RelMetadataQuery.splitCount(largeInput); + return RelMetadataQuery.instance().splitCount(largeInput); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java index 6669d32..f1037e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java @@ -84,8 +84,8 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { - double leftRCount = RelMetadataQuery.getRowCount(join.getLeft()); - double rightRCount = RelMetadataQuery.getRowCount(join.getRight()); + double leftRCount = RelMetadataQuery.instance().getRowCount(join.getLeft()); + double rightRCount = RelMetadataQuery.instance().getRowCount(join.getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java index 61a3a64..c0086f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java @@ -78,7 +78,7 @@ public RelOptCost getDefaultCost() { @Override public RelOptCost getScanCost(HiveTableScan ts) { - return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.getAverageRowSize(ts)); + return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts)); } @Override @@ -87,7 +87,7 @@ public RelOptCost getAggregateCost(HiveAggregate aggregate) { return HiveCost.FACTORY.makeZeroCost(); } else { // 1. Sum of input cardinalities - final Double rCount = RelMetadataQuery.getRowCount(aggregate.getInput()); + final Double rCount = RelMetadataQuery.instance().getRowCount(aggregate.getInput()); if (rCount == null) { return null; } @@ -96,7 +96,7 @@ public RelOptCost getAggregateCost(HiveAggregate aggregate) { // 3. IO cost = cost of writing intermediary results to local FS + // cost of reading from local FS for transferring to GBy + // cost of transferring map outputs to GBy operator - final Double rAverageSize = RelMetadataQuery.getAverageRowSize(aggregate.getInput()); + final Double rAverageSize = RelMetadataQuery.instance().getAverageRowSize(aggregate.getInput()); if (rAverageSize == null) { return null; } @@ -129,8 +129,8 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { // 1. Sum of input cardinalities - final Double leftRCount = RelMetadataQuery.getRowCount(join.getLeft()); - final Double rightRCount = RelMetadataQuery.getRowCount(join.getRight()); + final Double leftRCount = RelMetadataQuery.instance().getRowCount(join.getLeft()); + final Double rightRCount = RelMetadataQuery.instance().getRowCount(join.getRight()); if (leftRCount == null || rightRCount == null) { return null; } @@ -151,8 +151,8 @@ public RelOptCost getCost(HiveJoin join) { // 3. IO cost = cost of writing intermediary results to local FS + // cost of reading from local FS for transferring to join + // cost of transferring map outputs to Join operator - final Double leftRAverageSize = RelMetadataQuery.getAverageRowSize(join.getLeft()); - final Double rightRAverageSize = RelMetadataQuery.getAverageRowSize(join.getRight()); + final Double leftRAverageSize = RelMetadataQuery.instance().getAverageRowSize(join.getLeft()); + final Double rightRAverageSize = RelMetadataQuery.instance().getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null; } @@ -187,8 +187,8 @@ public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = - RelMetadataQuery.cumulativeMemoryWithinPhase(join); - final Integer splitCount = RelMetadataQuery.splitCount(join); + RelMetadataQuery.instance().cumulativeMemoryWithinPhase(join); + final Integer splitCount = RelMetadataQuery.instance().splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { @@ -239,8 +239,8 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { // 1. Sum of input cardinalities - final Double leftRCount = RelMetadataQuery.getRowCount(join.getLeft()); - final Double rightRCount = RelMetadataQuery.getRowCount(join.getRight()); + final Double leftRCount = RelMetadataQuery.instance().getRowCount(join.getLeft()); + final Double rightRCount = RelMetadataQuery.instance().getRowCount(join.getRight()); if (leftRCount == null || rightRCount == null) { return null; } @@ -251,7 +251,7 @@ public RelOptCost getCost(HiveJoin join) { add(leftRCount). add(rightRCount). build(); - ImmutableBitSet.Builder streamingBuilder = new ImmutableBitSet.Builder(); + ImmutableBitSet.Builder streamingBuilder = ImmutableBitSet.builder(); switch (join.getStreamingSide()) { case LEFT_RELATION: streamingBuilder.set(0); @@ -266,8 +266,8 @@ public RelOptCost getCost(HiveJoin join) { final double cpuCost = HiveAlgorithmsUtil.computeMapJoinCPUCost(cardinalities, streaming); // 3. IO cost = cost of transferring small tables to join node * // degree of parallelism - final Double leftRAverageSize = RelMetadataQuery.getAverageRowSize(join.getLeft()); - final Double rightRAverageSize = RelMetadataQuery.getAverageRowSize(join.getRight()); + final Double leftRAverageSize = RelMetadataQuery.instance().getAverageRowSize(join.getLeft()); + final Double rightRAverageSize = RelMetadataQuery.instance().getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null; } @@ -277,8 +277,8 @@ public RelOptCost getCost(HiveJoin join) { build(); JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezMapJoinAlgorithm.INSTANCE); - final int parallelism = RelMetadataQuery.splitCount(join) == null - ? 1 : RelMetadataQuery.splitCount(join); + final int parallelism = RelMetadataQuery.instance().splitCount(join) == null + ? 1 : RelMetadataQuery.instance().splitCount(join); join.setJoinAlgorithm(oldAlgo); final double ioCost = algoUtils.computeMapJoinIOCost(relationInfos, streaming, parallelism); // 4. Result @@ -322,7 +322,7 @@ public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { return null; } // If simple map join, the whole relation goes in memory - return RelMetadataQuery.cumulativeMemoryWithinPhase(inMemoryInput); + return RelMetadataQuery.instance().cumulativeMemoryWithinPhase(inMemoryInput); } @Override @@ -376,7 +376,7 @@ public boolean isExecutable(HiveJoin join) { // What we need is a way to get buckets not splits JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezBucketJoinAlgorithm.INSTANCE); - Integer buckets = RelMetadataQuery.splitCount(smallInput); + Integer buckets = RelMetadataQuery.instance().splitCount(smallInput); join.setJoinAlgorithm(oldAlgo); if (buckets == null) { @@ -388,7 +388,7 @@ public boolean isExecutable(HiveJoin join) { for (int i=0; i joinKeysInChildren = new ArrayList(); @@ -196,7 +196,7 @@ public ImmutableBitSet getSortedInputs() throws CalciteSemanticException { for (int i=0; i exps, Rel } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return RelMetadataQuery.getNonCumulativeCost(this); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { + return relMetadataQuery.getNonCumulativeCost(this); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java index 3558676..1227489 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java @@ -101,8 +101,8 @@ public void implement(Implementor implementor) { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return RelMetadataQuery.getNonCumulativeCost(this); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { + return relMetadataQuery.getNonCumulativeCost(this); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java index 5788805..a19d372 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java @@ -122,8 +122,8 @@ public HiveTableScan copy(RelDataType newRowtype) { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return RelMetadataQuery.getNonCumulativeCost(this); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { + return relMetadataQuery.getNonCumulativeCost(this); } @Override public RelWriter explainTerms(RelWriter pw) { @@ -146,8 +146,9 @@ public void implement(Implementor implementor) { } + //getRows will call estimateRowCount @Override - public double getRows() { + public double estimateRowCount(RelMetadataQuery mq) { return ((RelOptHiveTable) table).getRowCount(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java index 070c7ea..fea7711 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java @@ -121,7 +121,7 @@ public void onMatch(RelOptRuleCall call) { // Do the columns used by the join appear in the output of the aggregate? final ImmutableBitSet aggregateColumns = aggregate.getGroupSet(); final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, - RelMetadataQuery.getPulledUpPredicates(join).pulledUpPredicates); + RelMetadataQuery.instance().getPulledUpPredicates(join).pulledUpPredicates); final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition()); final boolean allColumnsInAggregate = @@ -179,7 +179,7 @@ public void onMatch(RelOptRuleCall call) { unique = true; } else { final Boolean unique0 = - RelMetadataQuery.areColumnsUnique(joinInput, belowAggregateKey); + RelMetadataQuery.instance().areColumnsUnique(joinInput, belowAggregateKey); unique = unique0 != null && unique0; } if (unique) { @@ -299,8 +299,8 @@ public Integer apply(Integer a0) { } // Make a cost based decision to pick cheaper plan - RelOptCost afterCost = RelMetadataQuery.getCumulativeCost(r); - RelOptCost beforeCost = RelMetadataQuery.getCumulativeCost(aggregate); + RelOptCost afterCost = RelMetadataQuery.instance().getCumulativeCost(r); + RelOptCost beforeCost = RelMetadataQuery.instance().getCumulativeCost(aggregate); if (afterCost.isLt(beforeCost)) { call.transformTo(r); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java index 7d7631b..7d4411a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java @@ -112,7 +112,7 @@ public void onMatch(RelOptRuleCall call) { // arguments then we can use a more efficient form. if ((nonDistinctCount == 0) && (argListSets.size() == 1)) { for (Integer arg : argListSets.iterator().next()) { - Set colOrigs = RelMetadataQuery.getColumnOrigins(aggregate, arg); + Set colOrigs = RelMetadataQuery.instance().getColumnOrigins(aggregate, arg); if (null != colOrigs) { for (RelColumnOrigin colOrig : colOrigs) { RelOptHiveTable hiveTbl = (RelOptHiveTable)colOrig.getOriginTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java index 703c8c6..7a28f21 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java @@ -78,7 +78,7 @@ public HiveJoinPushTransitivePredicatesRule(Class clazz, registry.registerVisited(this, join); } - RelOptPredicateList preds = RelMetadataQuery.getPulledUpPredicates(join); + RelOptPredicateList preds = RelMetadataQuery.instance().getPulledUpPredicates(join); RexBuilder rB = join.getCluster().getRexBuilder(); RelNode lChild = call.rel(1); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 50e139b..6958993 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -128,7 +128,7 @@ public FilterReduceExpressionsRule(Class filterClass, RexNode newConditionExp; boolean reduced; final RelOptPredicateList predicates = - RelMetadataQuery.getPulledUpPredicates(filter.getInput()); + RelMetadataQuery.instance().getPulledUpPredicates(filter.getInput()); if (reduceExpressions(filter, expList, predicates)) { assert expList.size() == 1; newConditionExp = expList.get(0); @@ -242,7 +242,7 @@ public boolean matches(RelOptRuleCall call) { registry.registerVisited(this, project); } final RelOptPredicateList predicates = - RelMetadataQuery.getPulledUpPredicates(project.getInput()); + RelMetadataQuery.instance().getPulledUpPredicates(project.getInput()); final List expList = Lists.newArrayList(project.getProjects()); if (reduceExpressions(project, expList, predicates)) { @@ -274,9 +274,9 @@ public JoinReduceExpressionsRule(Class joinClass, final List expList = Lists.newArrayList(join.getCondition()); final int fieldCount = join.getLeft().getRowType().getFieldCount(); final RelOptPredicateList leftPredicates = - RelMetadataQuery.getPulledUpPredicates(join.getLeft()); + RelMetadataQuery.instance().getPulledUpPredicates(join.getLeft()); final RelOptPredicateList rightPredicates = - RelMetadataQuery.getPulledUpPredicates(join.getRight()); + RelMetadataQuery.instance().getPulledUpPredicates(join.getRight()); final RelOptPredicateList predicates = leftPredicates.union(rightPredicates.shift(fieldCount)); if (!reduceExpressions(join, expList, predicates)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java index 0af60e8..2f2297d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java @@ -96,7 +96,7 @@ public boolean matches(RelOptRuleCall call) { // Finally, if we do not reduce the input size, we bail out final int offset = sortLimit.offset == null ? 0 : RexLiteral.intValue(sortLimit.offset); if (offset + RexLiteral.intValue(sortLimit.fetch) - >= RelMetadataQuery.getRowCount(reducedInput)) { + >= RelMetadataQuery.instance().getRowCount(reducedInput)) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java index 618c717..573b75a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java @@ -59,7 +59,7 @@ public boolean matches(RelOptRuleCall call) { // Finally, if we do not reduce the size input enough, we bail out int limit = RexLiteral.intValue(sortLimit.fetch); - Double rowCount = RelMetadataQuery.getRowCount(sortLimit.getInput()); + Double rowCount = RelMetadataQuery.instance().getRowCount(sortLimit.getInput()); if (rowCount != null && limit <= reductionProportion * rowCount && rowCount - limit >= reductionTuples) { return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java index 0ec8bf1..04b94c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java @@ -80,7 +80,7 @@ public void onMatch(RelOptRuleCall call) { final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset); for (RelNode input : union.getInputs()) { // If we do not reduce the input size, we bail out - if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.getRowCount(input)) { + if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.instance().getRowCount(input)) { finishPushSortPastUnion = false; // Here we do some query rewrite. We first get the new fetchRN, which is // a sum of offset and fetch. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java index c04060f..b533451 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java @@ -47,7 +47,7 @@ protected FilterSelectivityEstimator(RelNode childRel) { super(true); this.childRel = childRel; - this.childCardinality = RelMetadataQuery.getRowCount(childRel); + this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel); } public Double estimateSelectivity(RexNode predicate) { @@ -254,7 +254,7 @@ private Double getMaxNDV(RexCall call) { for (RexNode op : call.getOperands()) { if (op instanceof RexInputRef) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, RelMetadataQuery.instance(), ((RexInputRef) op).getIndex()); if (tmpNDV > maxNDV) maxNDV = tmpNDV; @@ -262,7 +262,7 @@ private Double getMaxNDV(RexCall call) { irv = new InputReferencedVisitor(); irv.apply(op); for (Integer childProjIndx : irv.inputPosReferenced) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, childProjIndx); + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, RelMetadataQuery.instance(),childProjIndx); if (tmpNDV > maxNDV) maxNDV = tmpNDV; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java index 84fa518..66bc148 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java @@ -24,6 +24,7 @@ import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdCollation; import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.BuiltInMethod; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelCollation; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; @@ -46,7 +47,7 @@ private HiveRelMdCollation() {} //~ Methods ---------------------------------------------------------------- - public ImmutableList collations(HiveAggregate aggregate) { + public ImmutableList collations(HiveAggregate aggregate, RelMetadataQuery relMetadataQuery) { // Compute collations ImmutableList.Builder collationListBuilder = new ImmutableList.Builder(); @@ -60,7 +61,7 @@ private HiveRelMdCollation() {} new HiveRelCollation(collationListBuilder.build()))); } - public ImmutableList collations(HiveJoin join) { + public ImmutableList collations(HiveJoin join, RelMetadataQuery relMetadataQuery) { return join.getCollation(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java index 1220401..03872b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java @@ -58,8 +58,8 @@ private HiveRelMdDistinctRowCount() { // Catch-all rule when none of the others apply. @Override - public Double getDistinctRowCount(RelNode rel, ImmutableBitSet groupKey, - RexNode predicate) { + public Double getDistinctRowCount(RelNode rel, RelMetadataQuery relMetadataQuery, + ImmutableBitSet groupKey, RexNode predicate) { if (rel instanceof HiveTableScan) { return getDistinctRowCount((HiveTableScan) rel, groupKey, predicate); } @@ -67,7 +67,7 @@ public Double getDistinctRowCount(RelNode rel, ImmutableBitSet groupKey, * For now use Calcite' default formulas for propagating NDVs up the Query * Tree. */ - return super.getDistinctRowCount(rel, groupKey, predicate); + return super.getDistinctRowCount(rel, relMetadataQuery, groupKey, predicate); } private Double getDistinctRowCount(HiveTableScan htRel, ImmutableBitSet groupKey, @@ -83,39 +83,39 @@ private Double getDistinctRowCount(HiveTableScan htRel, ImmutableBitSet groupKey return Math.min(noDistinctRows, htRel.getRows()); } - public static Double getDistinctRowCount(RelNode r, int indx) { + public static Double getDistinctRowCount(RelNode r, RelMetadataQuery relMetadataQuery, int indx) { ImmutableBitSet bitSetOfRqdProj = ImmutableBitSet.of(indx); - return RelMetadataQuery.getDistinctRowCount(r, bitSetOfRqdProj, r + return relMetadataQuery.getDistinctRowCount(r, bitSetOfRqdProj, r .getCluster().getRexBuilder().makeLiteral(true)); } @Override - public Double getDistinctRowCount(Join rel, ImmutableBitSet groupKey, + public Double getDistinctRowCount(Join rel, RelMetadataQuery relMetadataQuery, ImmutableBitSet groupKey, RexNode predicate) { if (rel instanceof HiveJoin) { HiveJoin hjRel = (HiveJoin) rel; //TODO: Improve this if (hjRel.isLeftSemiJoin()) { - return RelMetadataQuery.getDistinctRowCount(hjRel.getLeft(), groupKey, + return relMetadataQuery.getDistinctRowCount(hjRel.getLeft(), groupKey, rel.getCluster().getRexBuilder().makeLiteral(true)); } else { - return RelMdUtil.getJoinDistinctRowCount(rel, rel.getJoinType(), + return RelMdUtil.getJoinDistinctRowCount(relMetadataQuery, rel, rel.getJoinType(), groupKey, predicate, true); } } - return RelMetadataQuery.getDistinctRowCount(rel, groupKey, predicate); + return relMetadataQuery.getDistinctRowCount(rel, groupKey, predicate); } /* * Favor Broad Plans over Deep Plans. */ - public RelOptCost getCumulativeCost(HiveJoin rel) { - RelOptCost cost = RelMetadataQuery.getNonCumulativeCost(rel); + public RelOptCost getCumulativeCost(HiveJoin rel, RelMetadataQuery relMetadataQuery) { + RelOptCost cost = relMetadataQuery.getNonCumulativeCost(rel); List inputs = rel.getInputs(); RelOptCost maxICost = HiveCost.ZERO; for (RelNode input : inputs) { - RelOptCost iCost = RelMetadataQuery.getCumulativeCost(input); + RelOptCost iCost = relMetadataQuery.getCumulativeCost(input); if (maxICost.isLt(iCost)) { maxICost = iCost; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java index b83f240..bd58e5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdDistribution; import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.BuiltInMethod; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; @@ -35,8 +36,7 @@ ChainedRelMetadataProvider.of( ImmutableList.of( ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.DISTRIBUTION.method, new HiveRelMdDistribution()), - RelMdDistribution.SOURCE)); + BuiltInMethod.DISTRIBUTION.method, new HiveRelMdDistribution()))); //~ Constructors ----------------------------------------------------------- @@ -44,12 +44,12 @@ private HiveRelMdDistribution() {} //~ Methods ---------------------------------------------------------------- - public RelDistribution distribution(HiveAggregate aggregate) { + public RelDistribution distribution(HiveAggregate aggregate, RelMetadataQuery relMetadataQuery) { return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, aggregate.getGroupSet().asList()); } - public RelDistribution distribution(HiveJoin join) { + public RelDistribution distribution(HiveJoin join, RelMetadataQuery relMetadataQuery) { return join.getDistribution(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdMemory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdMemory.java index bea5943..8fb6e18 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdMemory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdMemory.java @@ -52,8 +52,8 @@ public Double memory(HiveTableScan tableScan) { } public Double memory(HiveAggregate aggregate) { - final Double avgRowSize = RelMetadataQuery.getAverageRowSize(aggregate.getInput()); - final Double rowCount = RelMetadataQuery.getRowCount(aggregate.getInput()); + final Double avgRowSize = RelMetadataQuery.instance().getAverageRowSize(aggregate.getInput()); + final Double rowCount = RelMetadataQuery.instance().getRowCount(aggregate.getInput()); if (avgRowSize == null || rowCount == null) { return null; } @@ -79,8 +79,8 @@ public Double memory(HiveProject project) { public Double memory(HiveSortLimit sort) { if (sort.getCollation() != RelCollations.EMPTY) { // It sorts - final Double avgRowSize = RelMetadataQuery.getAverageRowSize(sort.getInput()); - final Double rowCount = RelMetadataQuery.getRowCount(sort.getInput()); + final Double avgRowSize = RelMetadataQuery.instance().getAverageRowSize(sort.getInput()); + final Double rowCount = RelMetadataQuery.instance().getRowCount(sort.getInput()); if (avgRowSize == null || rowCount == null) { return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdParallelism.java index 2f51d3b..1e83519 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdParallelism.java @@ -81,7 +81,7 @@ public Integer splitCount(HiveTableScan scan) { } public Integer splitCount(RelNode rel) { - Boolean newPhase = RelMetadataQuery.isPhaseTransition(rel); + Boolean newPhase = RelMetadataQuery.instance().isPhaseTransition(rel); if (newPhase == null) { return null; @@ -95,15 +95,15 @@ public Integer splitCount(RelNode rel) { // We do not repartition: take number of splits from children Integer splitCount = 0; for (RelNode input : rel.getInputs()) { - splitCount += RelMetadataQuery.splitCount(input); + splitCount += RelMetadataQuery.instance().splitCount(input); } return splitCount; } public Integer splitCountRepartition(RelNode rel) { // We repartition: new number of splits - final Double averageRowSize = RelMetadataQuery.getAverageRowSize(rel); - final Double rowCount = RelMetadataQuery.getRowCount(rel); + final Double averageRowSize = RelMetadataQuery.instance().getAverageRowSize(rel); + final Double rowCount = RelMetadataQuery.instance().getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index b7244fd..22fe122 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -98,11 +98,10 @@ * * */ - @Override - public RelOptPredicateList getPredicates(Project project) { + public RelOptPredicateList getPredicates(Project project, RelMetadataQuery relMetadataQuery) { RelNode child = project.getInput(); final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); - RelOptPredicateList childInfo = RelMetadataQuery.getPulledUpPredicates(child); + RelOptPredicateList childInfo = relMetadataQuery.getPulledUpPredicates(child); List projectPullUpPredicates = new ArrayList(); HashMultimap inpIndxToOutIndxMap = HashMultimap.create(); @@ -150,14 +149,13 @@ public RelOptPredicateList getPredicates(Project project) { } /** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */ - @Override - public RelOptPredicateList getPredicates(Join join) { + public RelOptPredicateList getPredicates(Join join, RelMetadataQuery relMetadataQuery) { RexBuilder rB = join.getCluster().getRexBuilder(); RelNode left = join.getInput(0); RelNode right = join.getInput(1); - RelOptPredicateList leftInfo = RelMetadataQuery.getPulledUpPredicates(left); - RelOptPredicateList rightInfo = RelMetadataQuery.getPulledUpPredicates(right); + RelOptPredicateList leftInfo = relMetadataQuery.getPulledUpPredicates(left); + RelOptPredicateList rightInfo = relMetadataQuery.getPulledUpPredicates(right); HiveJoinConditionBasedPredicateInference jI = new HiveJoinConditionBasedPredicateInference(join, RexUtil.composeConjunction(rB, leftInfo.pulledUpPredicates, false), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java index caf8978..6dca50d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java @@ -75,7 +75,6 @@ public Double getRowCount(Join join) { return join.getRows(); } - @Override public Double getRowCount(SemiJoin rel) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(rel); if (pkfk != null) { @@ -86,12 +85,11 @@ public Double getRowCount(SemiJoin rel) { } return pkfk.fkInfo.rowCount * selectivity; } - return super.getRowCount(rel); + return super.getRowCount(rel, RelMetadataQuery.instance()); } - @Override public Double getRowCount(Sort rel) { - final Double rowCount = RelMetadataQuery.getRowCount(rel.getInput()); + final Double rowCount = RelMetadataQuery.instance().getRowCount(rel.getInput()); if (rowCount != null && rel.fetch != null) { final int offset = rel.offset == null ? 0 : RexLiteral.intValue(rel.offset); final int limit = RexLiteral.intValue(rel.fetch); @@ -237,8 +235,8 @@ public static PKFKRelationInfo analyzeJoinForPKFK(Join joinRel) { return null; } - double leftRowCount = RelMetadataQuery.getRowCount(left); - double rightRowCount = RelMetadataQuery.getRowCount(right); + double leftRowCount = RelMetadataQuery.instance().getRowCount(left); + double rightRowCount = RelMetadataQuery.instance().getRowCount(right); if (leftIsKey && rightIsKey) { if (rightRowCount < leftRowCount) { @@ -253,8 +251,8 @@ public static PKFKRelationInfo analyzeJoinForPKFK(Join joinRel) { pkSide == 0 ? left : right, pkSide == 0 ? leftColIdx : rightColIdx) : false; - double leftNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(left, lBitSet, leftPred) : -1; - double rightNDV = isPKSideSimpleTree ? RelMetadataQuery.getDistinctRowCount(right, rBitSet, rightPred) : -1; + double leftNDV = isPKSideSimpleTree ? RelMetadataQuery.instance().getDistinctRowCount(left, lBitSet, leftPred) : -1; + double rightNDV = isPKSideSimpleTree ? RelMetadataQuery.instance().getDistinctRowCount(right, rBitSet, rightPred) : -1; /* * If the ndv of the PK - FK side don't match, and the PK side is a filter @@ -318,7 +316,7 @@ private static double pkSelectivity(Join joinRel, boolean leftChild, } else { HiveTableScan tScan = HiveRelMdUniqueKeys.getTableScan(child, true); if (tScan != null) { - double tRowCount = RelMetadataQuery.getRowCount(tScan); + double tRowCount = RelMetadataQuery.instance().getRowCount(tScan); return childRowCount / tRowCount; } else { return 1.0; @@ -328,7 +326,7 @@ private static double pkSelectivity(Join joinRel, boolean leftChild, private static boolean isKey(ImmutableBitSet c, RelNode rel) { boolean isKey = false; - Set keys = RelMetadataQuery.getUniqueKeys(rel); + Set keys = RelMetadataQuery.instance().getUniqueKeys(rel); if (keys != null) { for (ImmutableBitSet key : keys) { if (key.equals(c)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index a0eb83d..3ee23a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -63,8 +63,8 @@ public Double getSelectivity(HiveJoin j, RexNode predicate) throws CalciteSemant return computeInnerJoinSelectivity(j, predicate); } else if (j.getJoinType().equals(JoinRelType.LEFT) || j.getJoinType().equals(JoinRelType.RIGHT)) { - double left = RelMetadataQuery.getRowCount(j.getLeft()); - double right = RelMetadataQuery.getRowCount(j.getRight()); + double left = RelMetadataQuery.instance().getRowCount(j.getLeft()); + double right = RelMetadataQuery.instance().getRowCount(j.getRight()); double product = left * right; double innerJoinSelectivity = computeInnerJoinSelectivity(j, predicate); if (j.getJoinType().equals(JoinRelType.LEFT)) { @@ -97,14 +97,14 @@ private Double computeInnerJoinSelectivity(HiveJoin j, RexNode predicate) throws // Join which are part of join keys for (Integer ljk : jpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) { colStatMapBuilder.put(ljk, - HiveRelMdDistinctRowCount.getDistinctRowCount(j.getLeft(), ljk)); + HiveRelMdDistinctRowCount.getDistinctRowCount(j.getLeft(), RelMetadataQuery.instance(), ljk)); } // 2. Update Col Stats Map with col stats for columns from right side of // Join which are part of join keys for (Integer rjk : jpi.getProjsFromRightPartOfJoinKeysInChildSchema()) { colStatMapBuilder.put(rjk + rightOffSet, - HiveRelMdDistinctRowCount.getDistinctRowCount(j.getRight(), rjk)); + HiveRelMdDistinctRowCount.getDistinctRowCount(j.getRight(), RelMetadataQuery.instance(), rjk)); } colStatMap = colStatMapBuilder.build(); @@ -116,11 +116,11 @@ private Double computeInnerJoinSelectivity(HiveJoin j, RexNode predicate) throws ndvCrossProduct = exponentialBackoff(peLst, colStatMap); if (j.isLeftSemiJoin()) - ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()), + ndvCrossProduct = Math.min(RelMetadataQuery.instance().getRowCount(j.getLeft()), ndvCrossProduct); else - ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()) - * RelMetadataQuery.getRowCount(j.getRight()), ndvCrossProduct); + ndvCrossProduct = Math.min(RelMetadataQuery.instance().getRowCount(j.getLeft()) + * RelMetadataQuery.instance().getRowCount(j.getRight()), ndvCrossProduct); } // 4. Join Selectivity = 1/NDV diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java index 3224039..9b4c89e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java @@ -81,10 +81,10 @@ private HiveRelMdSize() {} final RelNode left = rel.getLeft(); final RelNode right = rel.getRight(); final List lefts = - RelMetadataQuery.getAverageColumnSizes(left); + RelMetadataQuery.instance().getAverageColumnSizes(left); List rights = null; if (!rel.isLeftSemiJoin()) { - rights = RelMetadataQuery.getAverageColumnSizes(right); + rights = RelMetadataQuery.instance().getAverageColumnSizes(right); } if (lefts == null && rights == null) { return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 7c22c33..058e886 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -34,6 +34,7 @@ import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdUniqueKeys; import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BitSets; @@ -59,12 +60,12 @@ * Inferring Uniqueness for all columns is very expensive right now. The flip * side of doing this is, it only works post Field Trimming. */ - public Set getUniqueKeys(Project rel, boolean ignoreNulls) { + public Set getUniqueKeys(Project rel, RelMetadataQuery relMetadataQuery, boolean ignoreNulls) { HiveTableScan tScan = getTableScan(rel.getInput(), false); if ( tScan == null ) { - Function fn = RelMdUniqueKeys.SOURCE.apply( + Function fn = (Function) RelMdUniqueKeys.SOURCE.apply( rel.getClass(), BuiltInMetadata.UniqueKeys.class); return ((BuiltInMetadata.UniqueKeys) fn.apply(rel)) .getUniqueKeys(ignoreNulls); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java index 00f1acb..0dcd2ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java @@ -431,8 +431,8 @@ OpAttr visit(HiveSortLimit sortRel) throws SemanticException { // 1.a. Extract order for each column from collation // Generate sortCols and order - ImmutableBitSet.Builder sortColsPosBuilder = new ImmutableBitSet.Builder(); - ImmutableBitSet.Builder sortOutputColsPosBuilder = new ImmutableBitSet.Builder(); + ImmutableBitSet.Builder sortColsPosBuilder = ImmutableBitSet.builder(); + ImmutableBitSet.Builder sortOutputColsPosBuilder = ImmutableBitSet.builder(); Map obRefToCallMap = sortRel.getInputRefToCallMap(); List sortCols = new ArrayList(); StringBuilder order = new StringBuilder();