diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e251920d8b..efcdeda8c7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1648,6 +1648,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "tries to modify the original materialization contents to reflect the latest changes to the\n" + "materialized view source tables, instead of rebuilding the contents fully. Incremental rebuild\n" + "is based on the materialized view algebraic incremental rewriting."), + HIVE_MATERIALIZED_VIEW_ROW_ID_SELECTIVITY("hive.materializedview.writeid.filter.selectivity", 0.1f, + "Incremental rebuild introduces filter conditions on ROW__ID column. This parameter sets\n" + + "the selectivity of filter condition on ROW__ID to the given value for the materialized\n" + + "view rewriting cost model. Reducing this value can be useful to favour incremental rebuild\n" + + "over full rebuild."), HIVE_MATERIALIZED_VIEW_FILE_FORMAT("hive.materializedview.fileformat", "ORC", new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), "Default file format for CREATE MATERIALIZED VIEW statement"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java index 635d27e723..8911f6dfeb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveOnTezCostModel; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveRelMdCost; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdCollation; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdCumulativeCost; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistinctRowCount; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistribution; import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdMemory; @@ -67,6 +68,7 @@ public RelMetadataProvider getMetadataProvider() { return ChainedRelMetadataProvider.of(ImmutableList .of( HiveRelMdDistinctRowCount.SOURCE, + HiveRelMdCumulativeCost.SOURCE, new HiveRelMdCost(cm).getMetadataProvider(), HiveRelMdSelectivity.SOURCE, HiveRelMdRowCount.SOURCE, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveMaterializedViewsRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveMaterializedViewsRelMetadataProvider.java new file mode 100644 index 0000000000..47f03c9c24 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveMaterializedViewsRelMetadataProvider.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistinctRowCount; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdRowCount; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSelectivity; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSize; +import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdUniqueKeys; + +/** + * Metadata provider specifically created for materialized views (vs metadata provider + * used in other phases such as join reordering). + * This metadata provider takes some useful pieces from the default Hive provider + * (selectivity improvements or distinct row count) and combines them with some of the + * original Calcite providers, e.g., overall cost, since provider for cost in default + * Hive cost model is tailored towards join reordering, e.g., cost is accounted only + * for join operators and not other operators, etc. + */ +public class HiveMaterializedViewsRelMetadataProvider { + + private final HiveConf hiveConf; + + + public HiveMaterializedViewsRelMetadataProvider(HiveConf hiveConf) { + this.hiveConf = hiveConf; + } + + public RelMetadataProvider getMetadataProvider() { + // Get the heuristic selectivity for selectivity of condition over ROW__ID column + final double rowIdCondSelectivity = (double) HiveConf.getFloatVar( + this.hiveConf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_ROW_ID_SELECTIVITY); + + // Return MD provider + return ChainedRelMetadataProvider.of(ImmutableList + .of( + HiveRelMdDistinctRowCount.SOURCE, + new HiveRelMdSelectivity(rowIdCondSelectivity).getMetadataProvider(), + HiveRelMdRowCount.SOURCE, + HiveRelMdUniqueKeys.SOURCE, + HiveRelMdSize.SOURCE, + DefaultRelMetadataProvider.INSTANCE)); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java index 43f8508ffb..eeefe2edba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java @@ -36,21 +36,31 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FilterSelectivityEstimator extends RexVisitorImpl { + + private static final Logger LOG = LoggerFactory.getLogger(FilterSelectivityEstimator.class); + + private final RelNode childRel; - private final double childCardinality; + private final double childCardinality; + private final double rowIdCondSelectivity; private final RelMetadataQuery mq; - protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) { + protected FilterSelectivityEstimator(RelNode childRel, double rowIdCondSelectivity, + RelMetadataQuery mq) { super(true); this.mq = mq; this.childRel = childRel; this.childCardinality = mq.getRowCount(childRel); + this.rowIdCondSelectivity = rowIdCondSelectivity; } public Double estimateSelectivity(RexNode predicate) { @@ -70,6 +80,17 @@ public Double visitCall(RexCall call) { return 1.0; } + /* + * Heuristic for predicates on ROW__ID till we have statistics (including + * distribution) available for it. + */ + if (isRowIdFilteringPredicate(call, this.childRel)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Predicate on ROW__ID: " + call + " -> Adjusting estimated cost"); + } + return rowIdCondSelectivity; + } + Double selectivity = null; SqlKind op = getOp(call); @@ -277,18 +298,44 @@ private Double getMaxNDV(RexCall call) { private boolean isPartitionPredicate(RexNode expr, RelNode r) { if (r instanceof Project) { - expr = RelOptUtil.pushFilterPastProject(expr, (Project) r); - return isPartitionPredicate(expr, ((Project) r).getInput()); + Project projOp = (Project) r; + expr = RelOptUtil.pushPastProject(expr, projOp); + return isPartitionPredicate(expr, projOp.getInput()); } else if (r instanceof Filter) { return isPartitionPredicate(expr, ((Filter) r).getInput()); } else if (r instanceof HiveTableScan) { - RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScan) r).getTable(); + RelOptHiveTable table = (RelOptHiveTable) r.getTable(); ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } return false; } + private boolean isRowIdFilteringPredicate(RexNode expr, RelNode r) { + if (r instanceof Project) { + Project projOp = (Project) r; + expr = RelOptUtil.pushPastProject(expr, projOp); + return isRowIdFilteringPredicate(expr, projOp.getInput()); + } else if (r instanceof Filter) { + return isRowIdFilteringPredicate(expr, ((Filter) r).getInput()); + } else if (r instanceof HiveTableScan) { + RelOptHiveTable table = (RelOptHiveTable) r.getTable(); + ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr); + if (cols.isEmpty()) { + return false; + } + boolean isRowIdFilteringPredicate = true; + for (int pos : cols) { + String colName = table.getRowType().getFieldNames().get(pos); + if (!colName.equals(VirtualColumn.ROWID.getName())) { + isRowIdFilteringPredicate = false; + } + } + return isRowIdFilteringPredicate; + } + return false; + } + private SqlKind getOp(RexCall call) { SqlKind op = call.getKind(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCumulativeCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCumulativeCost.java new file mode 100644 index 0000000000..3dea1ce479 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCumulativeCost.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.stats; + +import java.util.List; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.metadata.BuiltInMetadata; +import org.apache.calcite.rel.metadata.BuiltInMetadata.CumulativeCost; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; +import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; + +public class HiveRelMdCumulativeCost + implements MetadataHandler { + + private static final HiveRelMdCumulativeCost INSTANCE = + new HiveRelMdCumulativeCost(); + + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.CUMULATIVE_COST.method, INSTANCE); + + + private HiveRelMdCumulativeCost() { + } + + @Override + public MetadataDef getDef() { + return BuiltInMetadata.CumulativeCost.DEF; + } + + /* + * Favor Broad Plans over Deep Plans. + */ + public RelOptCost getCumulativeCost(HiveJoin rel, RelMetadataQuery mq) { + RelOptCost cost = mq.getNonCumulativeCost(rel); + List inputs = rel.getInputs(); + RelOptCost maxICost = HiveCost.ZERO; + for (RelNode input : inputs) { + RelOptCost iCost = mq.getCumulativeCost(input); + if (maxICost.isLt(iCost)) { + maxICost = iCost; + } + } + return cost.plus(maxICost); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java index 80b939a9f6..7056e77330 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java @@ -23,7 +23,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.SemiJoin; -import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdDistinctRowCount; import org.apache.calcite.rel.metadata.RelMdUtil; @@ -38,21 +37,16 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; -import com.google.common.collect.ImmutableList; public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount { private static final HiveRelMdDistinctRowCount INSTANCE = new HiveRelMdDistinctRowCount(); - public static final RelMetadataProvider SOURCE = ChainedRelMetadataProvider - .of(ImmutableList.of( - + public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.DISTINCT_ROW_COUNT.method, INSTANCE), + BuiltInMethod.DISTINCT_ROW_COUNT.method, INSTANCE); - ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.CUMULATIVE_COST.method, INSTANCE))); private HiveRelMdDistinctRowCount() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index 575902d78d..c092124518 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -17,15 +17,18 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.SemiJoin; +import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdSelectivity; import org.apache.calcite.rel.metadata.RelMdUtil; @@ -34,6 +37,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinLeafPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; @@ -45,19 +49,29 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { - public static final RelMetadataProvider SOURCE = - ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.SELECTIVITY.method, new HiveRelMdSelectivity()); + private final double rowIdCondSelectivity; - //~ Constructors ----------------------------------------------------------- + public HiveRelMdSelectivity(double rowIdCondSelectivity) { + this.rowIdCondSelectivity = rowIdCondSelectivity; + } - private HiveRelMdSelectivity() {} + public RelMetadataProvider getMetadataProvider() { + return ChainedRelMetadataProvider.of( + ImmutableList.of( + ReflectiveRelMetadataProvider.reflectiveSource(this, + BuiltInMethod.SELECTIVITY.method), + RelMdSelectivity.SOURCE)); + } //~ Methods ---------------------------------------------------------------- + public Double getSelectivity(RelSubset rel, RelMetadataQuery mq, RexNode predicate) { + return mq.getSelectivity(Util.first(rel.getBest(), rel.getOriginal()), predicate); + } + public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) { if (predicate != null) { - FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, mq); + FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, rowIdCondSelectivity, mq); return filterSelEstmator.estimateSelectivity(predicate); } @@ -86,7 +100,7 @@ private Double computeInnerJoinSelectivity(Join j, RelMetadataQuery mq, RexNode getCombinedPredicateForJoin(j, predicate); if (!predInfo.getKey()) { return - new FilterSelectivityEstimator(j, mq). + new FilterSelectivityEstimator(j, rowIdCondSelectivity, mq). estimateSelectivity(predInfo.getValue()); } @@ -249,12 +263,6 @@ protected double exponentialBackoff(List peLst, /** * Compute Max NDV to determine Join Selectivity. - * - * @param jlpi - * @param colStatMap - * Immutable Map of Projection Index (in Join Schema) to Column Stat - * @param rightProjOffSet - * @return */ private static Double getMaxNDVForJoinSelectivity(JoinLeafPredicateInfo jlpi, ImmutableMap colStatMap) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java index 97097381d9..ede28d1869 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; @@ -27,8 +28,10 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableNullableList; +import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; @@ -51,6 +54,10 @@ private HiveRelMdSize() {} //~ Methods ---------------------------------------------------------------- + public List averageColumnSizes(RelSubset rel, RelMetadataQuery mq) { + return mq.getAverageColumnSizes(Util.first(rel.getBest(), rel.getOriginal())); + } + public List averageColumnSizes(HiveTableScan scan, RelMetadataQuery mq) { List neededcolsLst = scan.getNeededColIndxsFrmReloptHT(); List columnStatistics = ((RelOptHiveTable) scan.getTable()) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 3bf62c535c..d88f82897e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -26,6 +26,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; @@ -40,6 +41,7 @@ import org.apache.calcite.util.BitSets; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -54,6 +56,10 @@ return BuiltInMetadata.UniqueKeys.DEF; } + public Set getUniqueKeys(RelSubset rel, RelMetadataQuery mq, boolean ignoreNulls) { + return mq.getUniqueKeys(Util.first(rel.getBest(), rel.getOriginal()), ignoreNulls); + } + /* * Infer Uniquenes if: - rowCount(col) = ndv(col) - TBD for numerics: max(col) * - min(col) = rowCount(col) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 361f150193..afc0e1dc7b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -148,6 +148,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveMaterializedViewsRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOpMaterializationValidator; @@ -2224,9 +2225,11 @@ private RelNode copyNodeScan(RelNode scan) { if (!materializations.isEmpty()) { perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - // Use Calcite cost model for view rewriting + // Use materialized views specific cost model optCluster.invalidateMetadataQuery(); - RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(DefaultRelMetadataProvider.INSTANCE)); + RelMetadataQuery.THREAD_PROVIDERS.set( + JaninoRelMetadataProvider.of( + new HiveMaterializedViewsRelMetadataProvider(conf).getMetadataProvider())); // Add materializations to planner for (RelOptMaterialization materialization : materializations) { diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out index 3852945776..6d6dcc6887 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -486,18 +486,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((ROW__ID.writeid > 1) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) Statistics: Num rows: 501 Data size: 175536 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + predicate: ((ROW__ID.writeid > 1) and (UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -506,18 +506,18 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(value, 'hll') keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6306 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: may be used (ACID table) @@ -529,14 +529,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3153 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3153 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3153 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -550,7 +550,7 @@ STAGE PLANS: tables: partition: key - replace: true + replace: false table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -575,70 +575,8 @@ PREHOOK: Output: default@partition_mv_1 POSTHOOK: query: ALTER MATERIALIZED VIEW partition_mv_1 REBUILD POSTHOOK: type: QUERY POSTHOOK: Input: default@src_txn -POSTHOOK: Output: default@partition_mv_1@key=201 -POSTHOOK: Output: default@partition_mv_1@key=202 -POSTHOOK: Output: default@partition_mv_1@key=203 -POSTHOOK: Output: default@partition_mv_1@key=205 -POSTHOOK: Output: default@partition_mv_1@key=207 -POSTHOOK: Output: default@partition_mv_1@key=208 -POSTHOOK: Output: default@partition_mv_1@key=209 -POSTHOOK: Output: default@partition_mv_1@key=213 -POSTHOOK: Output: default@partition_mv_1@key=214 -POSTHOOK: Output: default@partition_mv_1@key=216 -POSTHOOK: Output: default@partition_mv_1@key=217 -POSTHOOK: Output: default@partition_mv_1@key=218 -POSTHOOK: Output: default@partition_mv_1@key=219 -POSTHOOK: Output: default@partition_mv_1@key=221 -POSTHOOK: Output: default@partition_mv_1@key=222 -POSTHOOK: Output: default@partition_mv_1@key=223 -POSTHOOK: Output: default@partition_mv_1@key=224 -POSTHOOK: Output: default@partition_mv_1@key=226 -POSTHOOK: Output: default@partition_mv_1@key=228 -POSTHOOK: Output: default@partition_mv_1@key=229 -POSTHOOK: Output: default@partition_mv_1@key=230 -POSTHOOK: Output: default@partition_mv_1@key=233 -POSTHOOK: Output: default@partition_mv_1@key=235 -POSTHOOK: Output: default@partition_mv_1@key=237 POSTHOOK: Output: default@partition_mv_1@key=238 -POSTHOOK: Output: default@partition_mv_1@key=239 -POSTHOOK: Output: default@partition_mv_1@key=241 -POSTHOOK: Output: default@partition_mv_1@key=242 -POSTHOOK: Output: default@partition_mv_1@key=244 -POSTHOOK: Output: default@partition_mv_1@key=247 -POSTHOOK: Output: default@partition_mv_1@key=248 -POSTHOOK: Output: default@partition_mv_1@key=249 -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: partition_mv_1 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_1 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: SELECT * FROM partition_mv_1 where key = 238 PREHOOK: type: QUERY PREHOOK: Input: default@partition_mv_1 @@ -859,20 +797,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src_txn - filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D) and (ROW__ID.writeid > 2)) (type: boolean) Statistics: Num rows: 502 Data size: 175904 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) - Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + predicate: ((ROW__ID.writeid > 2) and (UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 6307 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6307 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 6307 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -906,14 +844,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -922,18 +860,18 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: value, key - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(value, 'hll') keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 10597 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct) Reducer 3 Execution mode: llap @@ -943,14 +881,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 5298 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 5298 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 5298 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -964,7 +902,7 @@ STAGE PLANS: tables: partition: key - replace: true + replace: false table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -991,70 +929,8 @@ POSTHOOK: query: ALTER MATERIALIZED VIEW partition_mv_3 REBUILD POSTHOOK: type: QUERY POSTHOOK: Input: default@src_txn POSTHOOK: Input: default@src_txn_2 -POSTHOOK: Output: default@partition_mv_3@key=201 -POSTHOOK: Output: default@partition_mv_3@key=202 -POSTHOOK: Output: default@partition_mv_3@key=203 -POSTHOOK: Output: default@partition_mv_3@key=205 -POSTHOOK: Output: default@partition_mv_3@key=207 -POSTHOOK: Output: default@partition_mv_3@key=208 -POSTHOOK: Output: default@partition_mv_3@key=209 -POSTHOOK: Output: default@partition_mv_3@key=213 -POSTHOOK: Output: default@partition_mv_3@key=214 -POSTHOOK: Output: default@partition_mv_3@key=216 -POSTHOOK: Output: default@partition_mv_3@key=217 -POSTHOOK: Output: default@partition_mv_3@key=218 -POSTHOOK: Output: default@partition_mv_3@key=219 -POSTHOOK: Output: default@partition_mv_3@key=221 -POSTHOOK: Output: default@partition_mv_3@key=222 -POSTHOOK: Output: default@partition_mv_3@key=223 -POSTHOOK: Output: default@partition_mv_3@key=224 -POSTHOOK: Output: default@partition_mv_3@key=226 -POSTHOOK: Output: default@partition_mv_3@key=228 -POSTHOOK: Output: default@partition_mv_3@key=229 -POSTHOOK: Output: default@partition_mv_3@key=230 -POSTHOOK: Output: default@partition_mv_3@key=233 -POSTHOOK: Output: default@partition_mv_3@key=235 -POSTHOOK: Output: default@partition_mv_3@key=237 POSTHOOK: Output: default@partition_mv_3@key=238 -POSTHOOK: Output: default@partition_mv_3@key=239 -POSTHOOK: Output: default@partition_mv_3@key=241 -POSTHOOK: Output: default@partition_mv_3@key=242 -POSTHOOK: Output: default@partition_mv_3@key=244 -POSTHOOK: Output: default@partition_mv_3@key=247 -POSTHOOK: Output: default@partition_mv_3@key=248 -POSTHOOK: Output: default@partition_mv_3@key=249 -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: partition_mv_3 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: partition_mv_3 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: SELECT * FROM partition_mv_3 where key = 238 PREHOOK: type: QUERY PREHOOK: Input: default@partition_mv_3 @@ -1069,7 +945,7 @@ val_238 238 val_238 238 val_238 238 val_238 238 -val_238_n2 238 -val_238_n2 238 val_238_n 238 val_238_n 238 +val_238_n2 238 +val_238_n2 238 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild.q.out index 4d37d82b6e..a4de1a055f 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rebuild.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rebuild.q.out @@ -77,14 +77,16 @@ POSTHOOK: Input: default@mv_rebuild PREHOOK: query: alter materialized view mv_rebuild rebuild PREHOOK: type: QUERY PREHOOK: Input: default@basetable_rebuild +PREHOOK: Input: default@mv_rebuild PREHOOK: Output: default@mv_rebuild POSTHOOK: query: alter materialized view mv_rebuild rebuild POSTHOOK: type: QUERY POSTHOOK: Input: default@basetable_rebuild +POSTHOOK: Input: default@mv_rebuild POSTHOOK: Output: default@mv_rebuild -POSTHOOK: Lineage: mv_rebuild._c2 EXPRESSION [(basetable_rebuild)basetable_rebuild.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: mv_rebuild.a SIMPLE [(basetable_rebuild)basetable_rebuild.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: mv_rebuild.b SIMPLE [(basetable_rebuild)basetable_rebuild.FieldSchema(name:b, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv_rebuild._c2 EXPRESSION [(basetable_rebuild)basetable_rebuild.FieldSchema(name:a, type:int, comment:null), (mv_rebuild)default.mv_rebuild.FieldSchema(name:_c2, type:bigint, comment:null), ] +POSTHOOK: Lineage: mv_rebuild.a EXPRESSION [(basetable_rebuild)basetable_rebuild.FieldSchema(name:a, type:int, comment:null), (mv_rebuild)default.mv_rebuild.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: mv_rebuild.b EXPRESSION [(basetable_rebuild)basetable_rebuild.FieldSchema(name:b, type:varchar(256), comment:null), (mv_rebuild)default.mv_rebuild.FieldSchema(name:b, type:varchar(256), comment:null), ] PREHOOK: query: select * from mv_rebuild PREHOOK: type: QUERY PREHOOK: Input: default@mv_rebuild