diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index 5f34e8d..8428615 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -684,7 +684,8 @@ public String apply(RexNode r) { // Note: this is the last step, trying to avoid the expensive call to the metadata provider // if possible Set predicatesInSubtree = Sets.newHashSet(); - for (RexNode pred : RelMetadataQuery.instance().getPulledUpPredicates(inp).pulledUpPredicates) { + final RelMetadataQuery mq = inp.getCluster().getMetadataQuery(); + for (RexNode pred : mq.getPulledUpPredicates(inp).pulledUpPredicates) { predicatesInSubtree.add(pred.toString()); predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java index 28c7362..cb033a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java @@ -199,7 +199,8 @@ public double computeSMBMapJoinIOCost( } public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { - Double currentMemory = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(input); + final RelMetadataQuery mq = input.getCluster().getMetadataQuery(); + Double currentMemory = mq.cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; @@ -310,7 +311,7 @@ public static Double getJoinMemory(HiveJoin join) { public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation streamingSide) { Double memory = 0.0; - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.RIGHT_RELATION) { // Left side @@ -338,7 +339,7 @@ public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { @@ -358,7 +359,8 @@ public static Integer getSplitCountWithoutRepartition(HiveJoin join) { } else { return null; } - return RelMetadataQuery.instance().splitCount(largeInput); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); + return mq.splitCount(largeInput); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java index 11b6aa3..b04492f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java @@ -85,7 +85,7 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); double leftRCount = mq.getRowCount(join.getLeft()); double rightRCount = mq.getRowCount(join.getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java index 2dbfd8f..a2e656d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java @@ -87,7 +87,7 @@ public RelOptCost getAggregateCost(HiveAggregate aggregate) { if (aggregate.isBucketedInput()) { return HiveCost.FACTORY.makeZeroCost(); } else { - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery(); // 1. Sum of input cardinalities final Double rCount = mq.getRowCount(aggregate.getInput()); if (rCount == null) { @@ -130,7 +130,7 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); // 1. Sum of input cardinalities final Double leftRCount = mq.getRowCount(join.getLeft()); final Double rightRCount = mq.getRowCount(join.getRight()); @@ -189,9 +189,9 @@ public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); - final Double memoryWithinPhase = - RelMetadataQuery.instance().cumulativeMemoryWithinPhase(join); - final Integer splitCount = RelMetadataQuery.instance().splitCount(join); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); + final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); + final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { @@ -241,7 +241,7 @@ public boolean isExecutable(HiveJoin join) { @Override public RelOptCost getCost(HiveJoin join) { - RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); // 1. Sum of input cardinalities final Double leftRCount = mq.getRowCount(join.getLeft()); final Double rightRCount = mq.getRowCount(join.getRight()); @@ -332,7 +332,8 @@ public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { return null; } // If simple map join, the whole relation goes in memory - return RelMetadataQuery.instance().cumulativeMemoryWithinPhase(inMemoryInput); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); + return mq.cumulativeMemoryWithinPhase(inMemoryInput); } @Override @@ -363,6 +364,7 @@ public String toString() { @Override public boolean isExecutable(HiveJoin join) { + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory(); // Check streaming side @@ -386,7 +388,7 @@ public boolean isExecutable(HiveJoin join) { // What we need is a way to get buckets not splits JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezBucketJoinAlgorithm.INSTANCE); - Integer buckets = RelMetadataQuery.instance().splitCount(smallInput); + Integer buckets = mq.splitCount(smallInput); join.setJoinAlgorithm(oldAlgo); if (buckets == null) { @@ -398,7 +400,7 @@ public boolean isExecutable(HiveJoin join) { for (int i=0; i colOrigs = RelMetadataQuery.instance().getColumnOrigins(aggregate, arg); + Set colOrigs = mq.getColumnOrigins(aggregate, arg); if (null != colOrigs) { for (RelColumnOrigin colOrig : colOrigs) { RelOptHiveTable hiveTbl = (RelOptHiveTable)colOrig.getOriginTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java index 8772d59..a5794f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java @@ -107,7 +107,7 @@ public void onMatch(RelOptRuleCall call) { RexNode newCondition = condition.accept(new RelOptUtil.RexInputConverter(rexBuilder, origFields, input.getRowType().getFieldList(), adjustments)); if (setOp instanceof Union && setOp.all) { - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(input); if (predicates != null) { ImmutableList.Builder listBuilder = ImmutableList.builder(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java index 65a19e1..9cdc5e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java @@ -80,7 +80,7 @@ public HiveJoinPushTransitivePredicatesRule(Class clazz, public void onMatch(RelOptRuleCall call) { Join join = call.rel(0); - RelOptPredicateList preds = RelMetadataQuery.instance().getPulledUpPredicates(join); + RelOptPredicateList preds = call.getMetadataQuery().getPulledUpPredicates(join); HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); assert registry != null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 6efc731..9a5d8e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -109,7 +109,7 @@ public FilterReduceExpressionsRule(Class filterClass, Lists.newArrayList(filter.getCondition()); RexNode newConditionExp; boolean reduced; - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(filter.getInput()); if (reduceExpressions(filter, expList, predicates, true)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java index 0644f0c..6e900b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -82,7 +82,7 @@ public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); - final RelMetadataQuery metadataProvider = RelMetadataQuery.instance(); + final RelMetadataQuery metadataProvider = call.getMetadataQuery(); // 1. Recompose filter possibly by pulling out common elements from DNF // expressions diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index ef50d9d..0fd3217 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -2294,7 +2294,7 @@ public void onMatch(RelOptRuleCall call) { // The join filters out the nulls. So, it's ok if there are // nulls in the join keys. - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, right, rightJoinKeys)) { //SQL2REL_LOGGER.fine(rightJoinKeys.toString() @@ -2509,7 +2509,7 @@ public void onMatch(RelOptRuleCall call) { // The join filters out the nulls. So, it's ok if there are // nulls in the join keys. - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, left, correlatedInputRefJoinKeys)) { //SQL2REL_LOGGER.fine(correlatedJoinKeys.toString() @@ -2588,7 +2588,7 @@ public void onMatch(RelOptRuleCall call) { // leftInputRel contains unique keys // i.e. each row is distinct and can group by on all the left // fields - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); if (!RelMdUtil.areColumnsDefinitelyUnique(mq, left, allCols)) { //SQL2REL_LOGGER.fine("There are no unique keys for " + left); return; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveSqCountCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveSqCountCheck.java index 7f3a4a4..30010dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveSqCountCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveSqCountCheck.java @@ -111,7 +111,7 @@ private boolean isSqlCountCheck(final HiveFilter filter) { final int groupCount = aggregate.getGroupCount(); final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder(); - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(aggregate.getInput()); if (predicates == null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java index 2f2297d..c15224d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java @@ -95,8 +95,9 @@ public boolean matches(RelOptRuleCall call) { // Finally, if we do not reduce the input size, we bail out final int offset = sortLimit.offset == null ? 0 : RexLiteral.intValue(sortLimit.offset); + final RelMetadataQuery mq = call.getMetadataQuery(); if (offset + RexLiteral.intValue(sortLimit.fetch) - >= RelMetadataQuery.instance().getRowCount(reducedInput)) { + >= mq.getRowCount(reducedInput)) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java index 54874e3..5ae960f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java @@ -84,7 +84,7 @@ public void onMatch(RelOptRuleCall call) { } final RexBuilder rexBuilder = sort.getCluster().getRexBuilder(); - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(sort.getInput()); if (predicates == null) { return; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java index 573b75a..5d6aaf0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java @@ -59,7 +59,7 @@ public boolean matches(RelOptRuleCall call) { // Finally, if we do not reduce the size input enough, we bail out int limit = RexLiteral.intValue(sortLimit.fetch); - Double rowCount = RelMetadataQuery.instance().getRowCount(sortLimit.getInput()); + Double rowCount = call.getMetadataQuery().getRowCount(sortLimit.getInput()); if (rowCount != null && limit <= reductionProportion * rowCount && rowCount - limit >= reductionTuples) { return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java index 04b94c3..dd54bcb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortUnionReduceRule.java @@ -80,7 +80,7 @@ public void onMatch(RelOptRuleCall call) { final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset); for (RelNode input : union.getInputs()) { // If we do not reduce the input size, we bail out - if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.instance().getRowCount(input)) { + if (RexLiteral.intValue(sort.fetch) + offset < call.getMetadataQuery().getRowCount(input)) { finishPushSortPastUnion = false; // Here we do some query rewrite. We first get the new fetchRN, which is // a sum of offset and fetch. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java index 4f6c610..614f0fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java @@ -76,7 +76,7 @@ public void onMatch(RelOptRuleCall call) { } final RexBuilder rexBuilder = union.getCluster().getRexBuilder(); - final RelMetadataQuery mq = RelMetadataQuery.instance(); + final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(union); if (predicates == null) { return;