diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8d9b5a3194..edd950f9fb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1162,6 +1162,13 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal // materialized views HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING("hive.materializedview.rewriting", false, "Whether to try to rewrite queries using the materialized views enabled for rewriting"), + HIVE_MATERIALIZED_VIEW_REWRITING_SELECTION_STRATEGY("hive.materializedview.rewriting.strategy", "heuristic", + new StringSet("heuristic", "costbased"), + "The strategy that should be used to cost and select the materialized view rewriting. \n" + + " heuristic: Always try to select the plan using the materialized view if rewriting produced one," + + "choosing the plan with lower cost among possible plans containing a materialized view\n" + + " costbased: Fully cost-based strategy, always use plan with lower cost, independently on whether " + + "it uses a materialized view or not"), HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW("hive.materializedview.rewriting.time.window", "0s", new TimeValidator(TimeUnit.SECONDS), "Time window, specified in seconds, after which outdated materialized views become invalid for automatic query rewriting.\n" + "For instance, if a materialized view is created and afterwards one of its source tables is changed at " + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 5ad4406cef..d76c8b6f40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -58,6 +58,8 @@ import javax.jdo.JDODataStoreException; import org.apache.calcite.plan.RelOptMaterialization; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Project; import org.apache.commons.io.FilenameUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileChecksum; @@ -1365,8 +1367,14 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { HiveMaterializedViewsRegistry.get().getRewritingMaterializedView( dbName, materializedViewTable.getTableName()); if (materialization != null) { - RelOptHiveTable cachedMaterializedViewTable = - (RelOptHiveTable) materialization.tableRel.getTable(); + RelNode viewScan = materialization.tableRel; + RelOptHiveTable cachedMaterializedViewTable; + if (viewScan instanceof Project) { + // There is a Project on top (due to nullability) + cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getInput(0).getTable(); + } else { + cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getTable(); + } if (cachedMaterializedViewTable.getHiveTableMD().getCreateTime() == materializedViewTable.getCreateTime()) { // It is in the cache and up to date diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 3f73fd7fcc..53dc8ec197 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -377,9 +377,8 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); rowType = dtFactory.createStructType(druidColTypes, druidColNames); RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), - conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); - + rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), + conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals, null, null); @@ -390,8 +389,8 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable } else { // Build Hive Table Scan Rel RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), - conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); + rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), + conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java index b0f1a8dfaf..b9b0e9ecf3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java @@ -20,11 +20,19 @@ public class HiveConfPlannerContext{ private boolean isCorrelatedColumns; + private boolean heuristicMaterializationStrategy; - public HiveConfPlannerContext(boolean isCorrelatedColumns) { + public HiveConfPlannerContext(boolean isCorrelatedColumns, boolean heuristicMaterializationStrategy) { this.isCorrelatedColumns = isCorrelatedColumns; + this.heuristicMaterializationStrategy = heuristicMaterializationStrategy; } - public boolean getIsCorrelatedColumns() { return isCorrelatedColumns;} + public boolean getIsCorrelatedColumns() { + return isCorrelatedColumns; + } + + public boolean isHeuristicMaterializationStrategy() { + return heuristicMaterializationStrategy; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java index efd8a35699..e85a99e846 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite; +import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptCluster; @@ -27,10 +28,21 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.server.CalciteServerStatement; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlCountAggFunction; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlMinMaxAggFunction; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlSumAggFunction; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlSumEmptyIsZeroAggFunction; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; + +import java.util.HashMap; +import java.util.Map; /** @@ -107,4 +119,38 @@ public RelBuilder empty() { return this.push(sort); } + public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) { + switch (flag) { + case YEAR: + return HiveFloorDate.YEAR; + case QUARTER: + return HiveFloorDate.QUARTER; + case MONTH: + return HiveFloorDate.MONTH; + case DAY: + return HiveFloorDate.DAY; + case HOUR: + return HiveFloorDate.HOUR; + case MINUTE: + return HiveFloorDate.MINUTE; + case SECOND: + return HiveFloorDate.SECOND; + } + return SqlStdOperatorTable.FLOOR; + } + + public static SqlAggFunction getRollup(SqlAggFunction aggregation) { + if (aggregation instanceof HiveSqlSumAggFunction + || aggregation instanceof HiveSqlMinMaxAggFunction + || aggregation instanceof HiveSqlSumEmptyIsZeroAggFunction) { + return aggregation; + } + if (aggregation instanceof HiveSqlCountAggFunction) { + HiveSqlCountAggFunction countAgg = (HiveSqlCountAggFunction) aggregation; + return new HiveSqlSumEmptyIsZeroAggFunction(countAgg.isDistinct(), countAgg.getReturnTypeInference(), + countAgg.getOperandTypeInference(), countAgg.getOperandTypeChecker()); + } + return null; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index 88aedb6381..fbf2202679 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -18,13 +18,23 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.cost; +import com.google.common.collect.Multimap; import org.apache.calcite.adapter.druid.DruidQuery; +import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.ConventionTraitDef; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptCostImpl; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.commons.math3.util.FastMath; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; +import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules; /** @@ -37,9 +47,14 @@ public class HiveVolcanoPlanner extends VolcanoPlanner { private static final boolean ENABLE_COLLATION_TRAIT = true; + private final boolean isHeuristic; + private static final double FACTOR = 0.2d; + + /** Creates a HiveVolcanoPlanner. */ public HiveVolcanoPlanner(HivePlannerContext conf) { super(HiveCost.FACTORY, conf); + isHeuristic = conf.unwrap(HiveConfPlannerContext.class).isHeuristicMaterializationStrategy(); } public static RelOptPlanner createPlanner(HivePlannerContext conf) { @@ -72,4 +87,56 @@ public void registerClass(RelNode node) { } super.registerClass(node); } + + /** + * The method extends the logic of the super method to decrease + * the cost of the plan if it contains materialized views + * (heuristic). + */ + public RelOptCost getCost(RelNode rel, RelMetadataQuery mq) { + assert rel != null : "pre-condition: rel != null"; + if (rel instanceof RelSubset) { + return getCost(((RelSubset) rel).getBest(), mq); + } + if (rel.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) + == Convention.NONE) { + return costFactory.makeInfiniteCost(); + } + // We get the cost of the operator + RelOptCost cost = mq.getNonCumulativeCost(rel); + if (!costFactory.makeZeroCost().isLt(cost)) { + // cost must be positive, so nudge it + cost = costFactory.makeTinyCost(); + } + // If this operator has a materialized view below, + // we make its cost tiny and adjust the cost of its + // inputs + boolean usesMaterializedViews = false; + Multimap, RelNode> nodeTypes = + mq.getNodeTypes(rel); + for (RelNode scan : nodeTypes.get(TableScan.class)) { + if (((RelOptHiveTable) scan.getTable()).getHiveTableMD().isMaterializedView()) { + usesMaterializedViews = true; + break; + } + } + if (isHeuristic && usesMaterializedViews) { + cost = costFactory.makeTinyCost(); + for (RelNode input : rel.getInputs()) { + // If a child of this expression uses a materialized view, + // then we decrease its cost by a certain factor. This is + // useful for e.g. partial rewritings, where a part of plan + // does not use the materialization, but we still want to + // decrease its cost so it is chosen instead of the original + // plan + cost = cost.plus(getCost(input, mq).multiplyBy(FACTOR)); + } + } else { + // No materialized view or not heuristic approach, normal costing + for (RelNode input : rel.getInputs()) { + cost = cost.plus(getCost(input, mq)); + } + } + return cost; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java index 94a3bac1a7..5fda6f928b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java @@ -123,7 +123,7 @@ public RelNode copy(RelTraitSet traitSet, List inputs) { */ public HiveTableScan copy(RelDataType newRowtype) { return new HiveTableScan(getCluster(), getTraitSet(), ((RelOptHiveTable) table), this.tblAlias, this.concatQbIDAlias, - newRowtype, this.useQBIdInDigest, this.insideView); + newRowtype, this.useQBIdInDigest, this.insideView); } @Override public RelWriter explainTerms(RelWriter pw) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java index df9c1802c8..fb9672ae88 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java @@ -17,41 +17,214 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; +import com.google.common.collect.ImmutableList; +import org.apache.calcite.avatica.util.TimeUnitRange; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewProjectFilterRule; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewOnlyFilterRule; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewProjectJoinRule; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewOnlyJoinRule; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewProjectAggregateRule; import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewOnlyAggregateRule; +import org.apache.calcite.rel.rules.ProjectRemoveRule; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinProjectTransposeRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; + +import java.util.List; +import java.util.Map; /** * Enable join and aggregate materialized view rewriting */ public class HiveMaterializedViewRule { + /** + * This PROGRAM will be executed when there is a partial rewriting + * (using union operator) to pull up the projection expressions + * on top of the input that executes the modified query. The goal + * of the program is to expose all available expressions below + * the root of the plan. + */ + private static final HepProgram PROGRAM = new HepProgramBuilder() + .addRuleInstance(HiveExtractRelNodeRule.INSTANCE) + .addRuleInstance(HiveTableScanProjectInsert.INSTANCE) + .addRuleCollection( + ImmutableList.of( + HiveFilterProjectTransposeRule.INSTANCE, + HiveJoinProjectTransposeRule.BOTH_PROJECT, + HiveJoinProjectTransposeRule.LEFT_PROJECT, + HiveJoinProjectTransposeRule.RIGHT_PROJECT, + HiveProjectMergeRule.INSTANCE)) + .addRuleInstance(ProjectRemoveRule.INSTANCE) + .addRuleInstance(HiveRootJoinProjectInsert.INSTANCE) + .build(); + public static final MaterializedViewProjectFilterRule INSTANCE_PROJECT_FILTER = - new MaterializedViewProjectFilterRule(HiveRelFactories.HIVE_BUILDER, true, - null, false); + new MaterializedViewProjectFilterRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM, false); public static final MaterializedViewOnlyFilterRule INSTANCE_FILTER = - new MaterializedViewOnlyFilterRule(HiveRelFactories.HIVE_BUILDER, true, - null, false); + new MaterializedViewOnlyFilterRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM, false); public static final MaterializedViewProjectJoinRule INSTANCE_PROJECT_JOIN = - new MaterializedViewProjectJoinRule(HiveRelFactories.HIVE_BUILDER, true, - null, false); + new MaterializedViewProjectJoinRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM, false); public static final MaterializedViewOnlyJoinRule INSTANCE_JOIN = - new MaterializedViewOnlyJoinRule(HiveRelFactories.HIVE_BUILDER, true, - null, false); + new MaterializedViewOnlyJoinRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM, false); + + public static final HiveMaterializedViewProjectAggregateRule INSTANCE_PROJECT_AGGREGATE = + new HiveMaterializedViewProjectAggregateRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM); + + public static final HiveMaterializedViewOnlyAggregateRule INSTANCE_AGGREGATE = + new HiveMaterializedViewOnlyAggregateRule(HiveRelFactories.HIVE_BUILDER, + true, PROGRAM); + + + protected static class HiveMaterializedViewProjectAggregateRule extends MaterializedViewProjectAggregateRule { + public HiveMaterializedViewProjectAggregateRule( + RelBuilderFactory relBuilderFactory, boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) { + super(relBuilderFactory, generateUnionRewriting, unionRewritingPullProgram); + } + + @Override + protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) { + return HiveRelBuilder.getFloorSqlFunction(flag); + } + + @Override + public SqlAggFunction getRollup(SqlAggFunction aggregation) { + return HiveRelBuilder.getRollup(aggregation); + } + } + + protected static class HiveMaterializedViewOnlyAggregateRule extends MaterializedViewOnlyAggregateRule { + public HiveMaterializedViewOnlyAggregateRule( + RelBuilderFactory relBuilderFactory, boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) { + super(relBuilderFactory, generateUnionRewriting, unionRewritingPullProgram); + } + + @Override + protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) { + return HiveRelBuilder.getFloorSqlFunction(flag); + } + + @Override + public SqlAggFunction getRollup(SqlAggFunction aggregation) { + return HiveRelBuilder.getRollup(aggregation); + } + } + + /** + * This rule is used within the PROGRAM that rewrites the query for + * partial rewritings. Its goal is to extract the RelNode from the + * RelSubset node so the rest of the rules in the PROGRAM can be + * applied correctly. + */ + private static class HiveExtractRelNodeRule extends RelOptRule { + + private static final HiveExtractRelNodeRule INSTANCE = + new HiveExtractRelNodeRule(); + + private HiveExtractRelNodeRule() { + super(operand(RelSubset.class, any())); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final RelSubset rel = call.rel(0); + call.transformTo(Util.first(rel.getBest(), rel.getOriginal())); + } + } + + /** + * This rule inserts an identity Project operator on top of a TableScan. + * The rule is useful to pull-up the projection expressions during partial + * rewriting using Union operator, as we would like to have all those + * expressions available at the top of the input to insert Filter conditions + * if needed. + */ + private static class HiveTableScanProjectInsert extends RelOptRule { + + private static final HiveTableScanProjectInsert INSTANCE = + new HiveTableScanProjectInsert(); + + private HiveTableScanProjectInsert() { + super(operand(Filter.class, operand(TableScan.class, any())), + HiveRelFactories.HIVE_BUILDER, "HiveTableScanProjectInsert"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Filter fil = call.rel(0); + final TableScan rel = call.rel(1); + // Add identity + RelBuilder relBuilder = call.builder(); + relBuilder.push(rel); + List identityFields = relBuilder.fields( + ImmutableBitSet.range(0, rel.getRowType().getFieldCount()).asList()); + RelNode newRel = relBuilder + .project(identityFields, ImmutableList.of(), true) + .build(); + call.transformTo(fil.copy(fil.getTraitSet(), ImmutableList.of(newRel))); + } + + } + + /** + * This rule adds a Project operator on top of the root operator if it is a join. + * This is important to meet the requirements set by the rewriting rule with + * respect to the plan returned by the input program. + */ + private static class HiveRootJoinProjectInsert extends RelOptRule { + + private static final HiveRootJoinProjectInsert INSTANCE = + new HiveRootJoinProjectInsert(); - public static final MaterializedViewProjectAggregateRule INSTANCE_PROJECT_AGGREGATE = - new MaterializedViewProjectAggregateRule(HiveRelFactories.HIVE_BUILDER, true, - null); + private HiveRootJoinProjectInsert() { + super(operand(Join.class, any()), + HiveRelFactories.HIVE_BUILDER, "HiveRootJoinProjectInsert"); + } - public static final MaterializedViewOnlyAggregateRule INSTANCE_AGGREGATE = - new MaterializedViewOnlyAggregateRule(HiveRelFactories.HIVE_BUILDER, true, - null); + @Override + public void onMatch(RelOptRuleCall call) { + final Join join = call.rel(0); + final HepRelVertex root = (HepRelVertex) call.getPlanner().getRoot(); + if (root.getCurrentRel() != join) { + // Bail out + return; + } + // The join is the root, but we should always end up with a Project operator + // on top. We will add it. + RelBuilder relBuilder = call.builder(); + relBuilder.push(join); + List identityFields = relBuilder.fields( + ImmutableBitSet.range(0, join.getRowType().getFieldCount()).asList()); + relBuilder.project(identityFields, ImmutableList.of(), true); + call.transformTo(relBuilder.build()); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index 4dc48f4710..0b1fe74f05 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -71,7 +71,6 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 612deb8327..41de17fd46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -44,18 +44,14 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import org.antlr.runtime.ClassicToken; import org.antlr.runtime.CommonToken; import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; -import org.apache.calcite.adapter.druid.DirectOperatorConversion; import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.adapter.druid.DruidSchema; import org.apache.calcite.adapter.druid.DruidTable; -import org.apache.calcite.adapter.druid.ExtractOperatorConversion; -import org.apache.calcite.adapter.druid.FloorOperatorConversion; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; @@ -110,13 +106,11 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWindow; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.ArraySqlType; import org.apache.calcite.sql.type.SqlTypeName; @@ -168,11 +162,8 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveAlgorithmsConf; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveVolcanoPlanner; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveConcat; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; @@ -366,15 +357,18 @@ private static RelOptPlanner createPlanner( HiveRulesRegistry registry = new HiveRulesRegistry(); Properties calciteConfigProperties = new Properties(); calciteConfigProperties.setProperty( - CalciteConnectionProperty.TIME_ZONE.camelName(), - conf.getLocalTimeZone().getId()); + CalciteConnectionProperty.TIME_ZONE.camelName(), + conf.getLocalTimeZone().getId()); calciteConfigProperties.setProperty( - CalciteConnectionProperty.MATERIALIZATIONS_ENABLED.camelName(), - Boolean.FALSE.toString()); + CalciteConnectionProperty.MATERIALIZATIONS_ENABLED.camelName(), + Boolean.FALSE.toString()); CalciteConnectionConfig calciteConfig = new CalciteConnectionConfigImpl(calciteConfigProperties); boolean isCorrelatedColumns = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_CORRELATED_MULTI_KEY_JOINS); + boolean heuristicMaterializationStrategy = HiveConf.getVar(conf, + HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_SELECTION_STRATEGY).equals("heuristic"); HivePlannerContext confContext = new HivePlannerContext(algorithmsConf, registry, calciteConfig, - corrScalarRexSQWithAgg, scalarAggNoGbyNoWin, new HiveConfPlannerContext(isCorrelatedColumns)); + corrScalarRexSQWithAgg, scalarAggNoGbyNoWin, + new HiveConfPlannerContext(isCorrelatedColumns, heuristicMaterializationStrategy)); return HiveVolcanoPlanner.createPlanner(confContext); } @@ -1487,11 +1481,21 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu calciteGenPlan = HiveRelDecorrelator.decorrelateQuery(calciteGenPlan); LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calciteGenPlan)); + // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, mdProvider.getMetadataProvider(), executorProvider); - // 3. Apply join order optimizations: reordering MST algorithm + // 3. Materialized view based rewriting + // We disable it for CTAS and MV creation queries (trying to avoid any problem + // due to data freshness) + if (conf.getBoolVar(ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING) && + !getQB().isMaterializedView() && !ctx.isLoadingMaterializedView() && !getQB().isCTAS()) { + calcitePreCboPlan = applyMaterializedViewRewriting(planner, + calcitePreCboPlan, mdProvider.getMetadataProvider(), executorProvider); + } + + // 4. Apply join order optimizations: reordering MST algorithm // If join optimizations failed because of missing stats, we continue with // the rest of optimizations if (profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)) { @@ -1538,100 +1542,13 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu disableSemJoinReordering = false; } - // 4.1. Run other optimizations that do not need stats + // 5. Run other optimizations that do not need stats perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, - HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, HiveUnionMergeRule.INSTANCE, - HiveProjectMergeRule.INSTANCE_NO_FORCE, HiveJoinCommuteRule.INSTANCE); + HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, HiveUnionMergeRule.INSTANCE, + HiveAggregateProjectMergeRule.INSTANCE, HiveProjectMergeRule.INSTANCE_NO_FORCE, HiveJoinCommuteRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Optimizations without stats 1"); - // 5. Materialized view based rewriting - // We disable it for CTAS and MV creation queries (trying to avoid any problem - // due to data freshness) - if (conf.getBoolVar(ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING) && - !getQB().isMaterializedView() && !ctx.isLoadingMaterializedView() && !getQB().isCTAS()) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - // Use Calcite cost model for view rewriting - RelMetadataProvider calciteMdProvider = DefaultRelMetadataProvider.INSTANCE; - RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(calciteMdProvider)); - planner.registerMetadataProviders(Lists.newArrayList(calciteMdProvider)); - // Add views to planner - List materializations = new ArrayList<>(); - try { - materializations = Hive.get().getValidMaterializedViews(rewrittenRebuild); - // We need to use the current cluster for the scan operator on views, - // otherwise the planner will throw an Exception (different planners) - materializations = Lists.transform(materializations, - new Function() { - @Override - public RelOptMaterialization apply(RelOptMaterialization materialization) { - final RelNode viewScan = materialization.tableRel; - final RelNode newViewScan; - if (viewScan instanceof Project) { - // There is a Project on top (due to nullability) - final Project pq = (Project) viewScan; - newViewScan = HiveProject.create(optCluster, copyNodeScan(pq.getInput()), - pq.getChildExps(), pq.getRowType(), Collections. emptyList()); - } else { - newViewScan = copyNodeScan(viewScan); - } - return new RelOptMaterialization(newViewScan, materialization.queryRel, null, - materialization.qualifiedTableName); - } - - private RelNode copyNodeScan(RelNode scan) { - final RelNode newScan; - if (scan instanceof DruidQuery) { - final DruidQuery dq = (DruidQuery) scan; - // Ideally we should use HiveRelNode convention. However, since Volcano planner - // throws in that case because DruidQuery does not implement the interface, - // we set it as Bindable. Currently, we do not use convention in Hive, hence that - // should be fine. - // TODO: If we want to make use of convention (e.g., while directly generating operator - // tree instead of AST), this should be changed. - newScan = DruidQuery.create(optCluster, optCluster.traitSetOf(BindableConvention.INSTANCE), - scan.getTable(), dq.getDruidTable(), - ImmutableList.of(dq.getTableScan())); - } else { - newScan = new HiveTableScan(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), - (RelOptHiveTable) scan.getTable(), scan.getTable().getQualifiedName().get(0), - null, false, false); - } - return newScan; - } - } - ); - } catch (HiveException e) { - LOG.warn("Exception loading materialized views", e); - } - if (!materializations.isEmpty()) { - for (RelOptMaterialization materialization : materializations) { - planner.addMaterialization(materialization); - } - // Add view-based rewriting rules to planner - planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_FILTER); - planner.addRule(HiveMaterializedViewRule.INSTANCE_FILTER); - planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_JOIN); - planner.addRule(HiveMaterializedViewRule.INSTANCE_JOIN); - planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_AGGREGATE); - planner.addRule(HiveMaterializedViewRule.INSTANCE_AGGREGATE); - // Optimize plan - planner.setRoot(calciteOptimizedPlan); - calciteOptimizedPlan = planner.findBestExp(); - // Remove view-based rewriting rules from planner - planner.clear(); - } - // Restore default cost model - RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider.getMetadataProvider())); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: View-based rewriting"); - } - - // 4.2. Run other optimizations that do not need stats - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, - HepMatchOrder.BOTTOM_UP, HiveAggregateProjectMergeRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Optimizations without stats 2"); - // 6. Run aggregate-join transpose (cost based) // If it failed because of missing stats, we continue with // the rest of optimizations @@ -1919,6 +1836,97 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv return basePlan; } + private RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode basePlan, + RelMetadataProvider mdProvider, RexExecutor executorProvider) { + final RelOptCluster optCluster = basePlan.getCluster(); + final PerfLogger perfLogger = SessionState.getPerfLogger(); + + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + final RelNode calcitePreMVRewritingPlan = basePlan; + + // Add views to planner + List materializations = new ArrayList<>(); + try { + materializations = Hive.get().getValidMaterializedViews(rewrittenRebuild); + // We need to use the current cluster for the scan operator on views, + // otherwise the planner will throw an Exception (different planners) + materializations = Lists.transform(materializations, + new Function() { + @Override + public RelOptMaterialization apply(RelOptMaterialization materialization) { + final RelNode viewScan = materialization.tableRel; + final RelNode newViewScan; + if (viewScan instanceof Project) { + // There is a Project on top (due to nullability) + final Project pq = (Project) viewScan; + newViewScan = HiveProject.create(optCluster, copyNodeScan(pq.getInput()), + pq.getChildExps(), pq.getRowType(), Collections. emptyList()); + } else { + newViewScan = copyNodeScan(viewScan); + } + return new RelOptMaterialization(newViewScan, materialization.queryRel, null, + materialization.qualifiedTableName); + } + + private RelNode copyNodeScan(RelNode scan) { + final RelNode newScan; + if (scan instanceof DruidQuery) { + final DruidQuery dq = (DruidQuery) scan; + // Ideally we should use HiveRelNode convention. However, since Volcano planner + // throws in that case because DruidQuery does not implement the interface, + // we set it as Bindable. Currently, we do not use convention in Hive, hence that + // should be fine. + // TODO: If we want to make use of convention (e.g., while directly generating operator + // tree instead of AST), this should be changed. + newScan = DruidQuery.create(optCluster, optCluster.traitSetOf(BindableConvention.INSTANCE), + scan.getTable(), dq.getDruidTable(), + ImmutableList.of(dq.getTableScan())); + } else { + newScan = new HiveTableScan(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), + (RelOptHiveTable) scan.getTable(), scan.getTable().getQualifiedName().get(0), + null, false, false); + } + return newScan; + } + } + ); + } catch (HiveException e) { + LOG.warn("Exception loading materialized views", e); + } + if (!materializations.isEmpty()) { + // Use Calcite cost model for view rewriting + optCluster.invalidateMetadataQuery(); + RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(DefaultRelMetadataProvider.INSTANCE)); + + // Add materializations to planner + for (RelOptMaterialization materialization : materializations) { + planner.addMaterialization(materialization); + } + // Add view-based rewriting rules to planner + planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_FILTER); + planner.addRule(HiveMaterializedViewRule.INSTANCE_FILTER); + planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_JOIN); + planner.addRule(HiveMaterializedViewRule.INSTANCE_JOIN); + planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_AGGREGATE); + planner.addRule(HiveMaterializedViewRule.INSTANCE_AGGREGATE); + // Optimize plan + planner.setRoot(basePlan); + basePlan = planner.findBestExp(); + // Remove view-based rewriting rules from planner + planner.clear(); + + // Restore default cost model + optCluster.invalidateMetadataQuery(); + RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: View-based rewriting"); + if (calcitePreMVRewritingPlan != basePlan) { + // Now we trigger some needed optimization rules again + basePlan = applyPreJoinOrderingTransforms(basePlan, mdProvider, executorProvider); + } + } + return basePlan; + } + /** * Run the HEP Planner with the given rule set. * @@ -2508,17 +2516,16 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); rowType = dtFactory.createStructType(druidColTypes, druidColNames); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), - dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, - intervals, null, null); + dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, + intervals, null, null); RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, - partitionCache, colStatsCache, noColsMissingStats); + rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, + partitionCache, colStatsCache, noColsMissingStats); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() - || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); - // Default Druid Standard + optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, + getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() + || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); } else { @@ -2533,8 +2540,8 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc fullyQualifiedTabName = tabMetaData.getTableName(); } RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, - partitionCache, colStatsCache, noColsMissingStats); + rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, + partitionCache, colStatsCache, noColsMissingStats); // Build Hive Table Scan Rel tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_1.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_1.q new file mode 100644 index 0000000000..fa604759a3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_1.q @@ -0,0 +1,183 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +-- EXAMPLE 1 +create materialized view mv1 enable rewrite as +select * from emps where empid < 150; +analyze table mv1 compute statistics for columns; + +explain +select * +from (select * from emps where empid < 120) t +join depts using (deptno); + +select * +from (select * from emps where empid < 120) t +join depts using (deptno); + +drop materialized view mv1; + +-- EXAMPLE 2 +create materialized view mv1 enable rewrite as +select deptno, name, salary, commission +from emps; +analyze table mv1 compute statistics for columns; + +explain +select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno); + +select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno); + +drop materialized view mv1; + +-- EXAMPLE 3 +create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno); +analyze table mv1 compute statistics for columns; + +explain +select empid deptno from emps +join depts using (deptno) where empid = 1; + +select empid deptno from emps +join depts using (deptno) where empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 4 +create materialized view mv1 enable rewrite as +select * from emps where empid < 200; +analyze table mv1 compute statistics for columns; + +explain +select * from emps where empid > 120 +union all select * from emps where empid < 150; + +select * from emps where empid > 120 +union all select * from emps where empid < 150; + +drop materialized view mv1; + +-- EXAMPLE 5 - NO MV, ALREADY UNIQUE +create materialized view mv1 enable rewrite as +select empid, deptno from emps group by empid, deptno; +analyze table mv1 compute statistics for columns; + +explain +select empid, deptno from emps group by empid, deptno; + +select empid, deptno from emps group by empid, deptno; + +drop materialized view mv1; + +-- EXAMPLE 5 - NO MV, ALREADY UNIQUE +create materialized view mv1 enable rewrite as +select empid, name from emps group by empid, name; +analyze table mv1 compute statistics for columns; + +explain +select empid, name from emps group by empid, name; + +select empid, name from emps group by empid, name; + +drop materialized view mv1; + +-- EXAMPLE 5 +create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select name, salary from emps group by name, salary; + +select name, salary from emps group by name, salary; + +drop materialized view mv1; + +-- EXAMPLE 6 +create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select name from emps group by name; + +select name from emps group by name; + +drop materialized view mv1; + +-- EXAMPLE 7 +create materialized view mv1 enable rewrite as +select name, salary from emps where deptno = 10 group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select name from emps where deptno = 10 group by name; + +select name from emps where deptno = 10 group by name; + +drop materialized view mv1; + +-- EXAMPLE 9 +create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select name from emps group by name; + +select name from emps group by name; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_2.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_2.q new file mode 100644 index 0000000000..16883884c4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_2.q @@ -0,0 +1,163 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +-- EXAMPLE 16 +create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where depts.deptno > 10 +group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 17 +create materialized view mv1 enable rewrite as +select depts.deptno, empid from depts +join emps using (deptno) where depts.deptno > 10 +group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 18 +create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where emps.deptno > 10 +group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 19 +create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid; +analyze table mv1 compute statistics for columns; + +explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid; + +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid; + +drop materialized view mv1; + +-- EXAMPLE 20 +create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid; +analyze table mv1 compute statistics for columns; + +explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno; + +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 23 +create materialized view mv1 enable rewrite as +select depts.name, dependents.name as name2, emps.deptno, depts.deptno as deptno2, dependents.empid +from depts, dependents, emps +where depts.deptno > 10 +group by depts.name, dependents.name, emps.deptno, depts.deptno, dependents.empid; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid; + +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_3.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_3.q new file mode 100644 index 0000000000..902e6169fd --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_3.q @@ -0,0 +1,114 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +-- EXAMPLE 34 +create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno); +analyze table mv1 compute statistics for columns; + +explain +select empid deptno from emps +join depts using (deptno) where empid = 1; + +select empid deptno from emps +join depts using (deptno) where empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 35 +create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno); +analyze table mv1 compute statistics for columns; + +explain +select empid deptno from emps +join depts using (deptno) where empid > 1; + +select empid deptno from emps +join depts using (deptno) where empid > 1; + +drop materialized view mv1; + +-- EXAMPLE 36 +create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno); +analyze table mv1 compute statistics for columns; + +explain +select empid deptno from emps +join depts using (deptno) where empid = 1; + +select empid deptno from emps +join depts using (deptno) where empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 38 +create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno); +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno); + +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno); + +drop materialized view mv1; + diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_4.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_4.q new file mode 100644 index 0000000000..2c0fadb502 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_4.q @@ -0,0 +1,177 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + + +-- EXAMPLE 10 +create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select name, count(*) as c, sum(empid) as s +from emps group by name; + +select name, count(*) as c, sum(empid) as s +from emps group by name; + +drop materialized view mv1; + +-- EXAMPLE 11 +create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary; +analyze table mv1 compute statistics for columns; + +explain +select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary; + +select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary; + +drop materialized view mv1; + +-- EXAMPLE 25 +create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, emps.deptno; +analyze table mv1 compute statistics for columns; + +explain +select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno; + +select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 27 +create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno >= 10 group by empid, emps.deptno; +analyze table mv1 compute statistics for columns; + +explain +select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno; + +select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 28 +create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) + 1 as c, sum(empid) as s +from emps join depts using (deptno) +where depts.deptno >= 10 group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno; + +select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 29 +create materialized view mv1 enable rewrite as +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +group by depts.name; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid; + +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid; + +drop materialized view mv1; + +-- EXAMPLE 32 +create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno; +analyze table mv1 compute statistics for columns; + +explain +select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno; + +select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_5.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_5.q new file mode 100644 index 0000000000..fc4445c764 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_5.q @@ -0,0 +1,296 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely; +alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely; + + +-- EXAMPLE 8 +create materialized view mv1 enable rewrite as +select name, deptno, salary from emps where deptno > 15 group by name, deptno, salary; +analyze table mv1 compute statistics for columns; + +explain +select name from emps where deptno >= 20 group by name; + +select name from emps where deptno >= 20 group by name; + +drop materialized view mv1; + +-- EXAMPLE 12 +create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary; +analyze table mv1 compute statistics for columns; + +explain +select name, sum(empid) as s +from emps where deptno > 15 group by name; + +select name, sum(empid) as s +from emps where deptno > 15 group by name; + +drop materialized view mv1; + +-- EXAMPLE 22 +create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by depts.deptno, dependents.empid; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid; + +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid; + +drop materialized view mv1; + +-- EXAMPLE 24 +create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select deptno from emps group by deptno; + +select deptno from emps group by deptno; + +drop materialized view mv1; + +-- EXAMPLE 26 +create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno; +analyze table mv1 compute statistics for columns; + +explain +select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno; + +select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno; + +drop materialized view mv1; + +-- EXAMPLE 30 +create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid; + +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid; + +drop materialized view mv1; + +-- EXAMPLE 31 +create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno; +analyze table mv1 compute statistics for columns; + +explain +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name; + +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name; + +drop materialized view mv1; + +-- EXAMPLE 41 +create materialized view mv1 enable rewrite as +select a.empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid); +analyze table mv1 compute statistics for columns; + +explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid); + +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid); + +drop materialized view mv1; + +-- EXAMPLE 42 +create materialized view mv1 enable rewrite as +select a.empid, a.deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid); +analyze table mv1 compute statistics for columns; + +explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid); + +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid); + +drop materialized view mv1; + +-- EXAMPLE 43 +create materialized view mv1 enable rewrite as +select empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno); +analyze table mv1 compute statistics for columns; + +explain +select empid from emps where empid = 1; + +select empid from emps where empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 44 +create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; +analyze table mv1 compute statistics for columns; + +explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; + +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 45a +create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; +analyze table mv1 compute statistics for columns; + +explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; + +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1; + +drop materialized view mv1; + +-- EXAMPLE 45b +create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian'; +analyze table mv1 compute statistics for columns; + +explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian'; + +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian'; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_6.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_6.q new file mode 100644 index 0000000000..cc844c5a4a --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_6.q @@ -0,0 +1,145 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely; +alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely; + + +-- EXAMPLE 13 +create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 10 group by name, deptno, salary; +analyze table mv1 compute statistics for columns; + +explain +select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary; + +select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary; + +drop materialized view mv1; + +-- EXAMPLE 14 +create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary; +analyze table mv1 compute statistics for columns; + +explain +select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary; + +select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary; + +drop materialized view mv1; + +-- EXAMPLE 37 +create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno); +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name); + +select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name); + +drop materialized view mv1; + +-- EXAMPLE 39 +create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno); +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno); + +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno); + +drop materialized view mv1; + +-- EXAMPLE 46 +create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno, emps.name as name1, emps.salary, emps.commission, dependents.name as name2 +from emps join dependents on (emps.empid = dependents.empid); +analyze table mv1 compute statistics for columns; + +explain +select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill'; + +select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill'; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_7.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_7.q new file mode 100644 index 0000000000..66ff561ccd --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_7.q @@ -0,0 +1,137 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250); +analyze table emps compute statistics for columns; + +create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts compute statistics for columns; + +create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents values (10, 'Michael'), (10, 'Jane'); +analyze table dependents compute statistics for columns; + +create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations compute statistics for columns; + +alter table emps add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely; +alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely; + +alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely; +alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely; + + +-- EXAMPLE 21 -- WORKS NOW +create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 +group by depts.deptno, dependents.empid; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno; + +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno; + +drop materialized view mv1; + +-- EXAMPLE 33 +create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid, count(emps.salary) as s +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by depts.deptno, dependents.empid; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid; + +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid; + +drop materialized view mv1; + +-- EXAMPLE 40 -- REWRITING HAPPENS BUT DISCARDED +-- DUE TO COST EXCEPT WITH HEURISTICS +create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno >= 10; +analyze table mv1 compute statistics for columns; + +explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0; + +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0; + +drop materialized view mv1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_8.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_8.q new file mode 100644 index 0000000000..837bf5b66b --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_8.q @@ -0,0 +1,116 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table if not exists source_table_001 ( +MY_DATE date, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true"); +insert into table source_table_001 + values ('2010-10-10', 1, 1, 'env', 1, 1); +analyze table source_table_001 compute statistics for columns; + +CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; +analyze table source_table_001_mv compute statistics for columns; + + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +LIMIT 100; + +explain +select +1, +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; + +explain +select +SUM(A.DOWN_VOLUME) + 0 AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; + +-- DOES NOT WORK - PROBLEM IN FIELD TRIMMER WITH OBY +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +ORDER BY A.MY_ID2 +LIMIT 100; + +-- WORKS WITH COLUMN STATS, CBO FAILS WITHOUT +explain +select +distinct A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A; + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; + +explain +select +SUM(A.DOWN_VOLUME) + SUM(A.UP_VOLUME) AS TOTAL_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10'); + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +TO_DATE('2010-01-10') +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10'); + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_DATE; + +drop materialized view source_table_001_mv; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_9.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_9.q new file mode 100644 index 0000000000..504b15f24a --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_9.q @@ -0,0 +1,71 @@ +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table if not exists source_table_001 ( +MY_DATE timestamp, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true"); +insert into table source_table_001 + values ('2010-10-10 00:00:00', 1, 1, 'env', 1, 1); +analyze table source_table_001 compute statistics for columns; + +CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE; +analyze table source_table_001_mv compute statistics for columns; + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour); + +DROP MATERIALIZED VIEW source_table_001_mv; + +CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour); +analyze table source_table_001_mv compute statistics for columns; + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to day),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to day); + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour); + +explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to second),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to second); + +DROP MATERIALIZED VIEW source_table_001_mv; diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out index 97f6d84480..a87040aeb1 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out @@ -511,7 +511,7 @@ rawDataSize 0 storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler totalSize 0 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[8][tables = [cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2, $hdt$_0]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( (SELECT a, c FROM cmv_basetable WHERE a = 3) table1 @@ -539,28 +539,35 @@ STAGE PLANS: #### A masked pattern was here #### Vertices: Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 30 Data size: 480 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((a = 3) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c (type: double) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Map 3 Map Operator Tree: TableScan alias: cmv_mat_view2 properties: - druid.fieldNames vc,a,b,c,userid - druid.fieldTypes timestamp with local time zone,int,varchar(256),double,varchar(256) - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc","a","b","c","userid"],"resultFormat":"compactedList"} + druid.fieldNames a,c + druid.fieldTypes int,double + druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} druid.query.type scan Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE value expressions: a (type: int), c (type: double) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 30 Data size: 480 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 30 Data size: 480 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: double), d (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -569,22 +576,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col1, _col3, _col9, _col12, _col13 - Statistics: Num rows: 90 Data size: 2610 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((_col13 = 3) and (_col9 = 3)) (type: boolean) - Statistics: Num rows: 22 Data size: 638 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col3 (type: double), _col1 (type: int), _col12 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 22 Data size: 638 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 22 Data size: 638 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + outputColumnNames: _col0, _col1, _col6 + Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col6 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -592,7 +596,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[8][tables = [cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2, $hdt$_0]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT * FROM ( (SELECT a, c FROM cmv_basetable WHERE a = 3) table1 JOIN diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out index 4da3d0930f..18eb1d1daa 100644 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out @@ -120,7 +120,7 @@ STAGE PLANS: alias: default.cmv_mat_view2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: decimal(10,2)) + expressions: 3 (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE ListSink @@ -266,7 +266,7 @@ POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE POSTHOOK: Input: default@cmv_mat_view2 POSTHOOK: Output: default@cmv_mat_view2 -Warning: Shuffle Join JOIN[4][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from ( (select a, c from cmv_basetable where a = 3) table1 @@ -292,17 +292,28 @@ STAGE PLANS: TableScan alias: default.cmv_mat_view2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: decimal(10,2)) + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(10,2)) TableScan alias: cmv_basetable Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + Filter Operator + predicate: ((a = 3) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(10,2)) Reduce Operator Tree: Join Operator condition map: @@ -310,22 +321,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col5, _col7, _col8 - Statistics: Num rows: 10 Data size: 3580 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((_col5 = 3) and (_col8 = 3)) (type: boolean) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col2 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col0 (type: int), _col7 (type: decimal(10,2)) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -333,7 +341,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[4][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from ( (select a, c from cmv_basetable where a = 3) table1 join diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out index d7ee468b49..32b408f5e4 100644 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out +++ b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out @@ -156,7 +156,7 @@ STAGE PLANS: alias: db2.cmv_mat_view2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: decimal(10,2)) + expressions: 3 (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out new file mode 100644 index 0000000000..75e828fd42 --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out @@ -0,0 +1,1104 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1 enable rewrite as +select * from emps where empid < 150 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select * from emps where empid < 150 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select * +from (select * from emps where empid < 120) t +join depts using (deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * +from (select * from emps where empid < 120) t +join depts using (deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((empid < 120) and deptno is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int) + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)), locationid (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)), _col2 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: int), _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int), _col6 (type: varchar(256)), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * +from (select * from emps where empid < 120) t +join depts using (deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select * +from (select * from emps where empid < 120) t +join depts using (deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 100 Bill 10000.0 1000 Sales 10 +10 110 Bill 10000.0 250 Sales 10 +10 110 Theodore 10000.0 250 Sales 10 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select deptno, name, salary, commission +from emps +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select deptno, name, salary, commission +from emps +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: deptno is not null (type: boolean) + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.name, emps.salary, emps.commission +from emps +join depts using (deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill 10000.0 1000 +Bill 10000.0 250 +Eric 8000.0 500 +Sebastian 7000.0 NULL +Theodore 10000.0 250 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select * from emps where empid < 200 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select * from emps where empid < 200 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select * from emps where empid > 120 +union all select * from emps where empid < 150 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from emps where empid > 120 +union all select * from emps where empid < 150 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 120) (type: boolean) + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid < 150) (type: boolean) + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from emps where empid > 120 +union all select * from emps where empid < 150 +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from emps where empid > 120 +union all select * from emps where empid < 150 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +100 10 Bill 10000.0 1000 +110 10 Bill 10000.0 250 +110 10 Theodore 10000.0 250 +150 10 Sebastian 7000.0 NULL +200 20 Eric 8000.0 500 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, deptno from emps group by empid, deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, deptno from emps group by empid, deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid, deptno from emps group by empid, deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid, deptno from emps group by empid, deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select empid, deptno from emps group by empid, deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +#### A masked pattern was here #### +POSTHOOK: query: select empid, deptno from emps group by empid, deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +#### A masked pattern was here #### +100 10 +110 10 +110 10 +150 10 +200 20 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, name from emps group by empid, name +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, name from emps group by empid, name +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid, name from emps group by empid, name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid, name from emps group by empid, name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 470 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 470 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select empid, name from emps group by empid, name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +#### A masked pattern was here #### +POSTHOOK: query: select empid, name from emps group by empid, name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +#### A masked pattern was here #### +100 Bill +110 Bill +110 Theodore +150 Sebastian +200 Eric +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name, salary from emps group by name, salary +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name, salary from emps group by name, salary +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), salary (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select name, salary from emps group by name, salary +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name, salary from emps group by name, salary +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill 10000.0 +Eric 8000.0 +Sebastian 7000.0 +Theodore 10000.0 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name from emps group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill +Eric +Sebastian +Theodore +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps where deptno = 10 group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary from emps where deptno = 10 group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps where deptno = 10 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps where deptno = 10 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps where deptno = 10 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name from emps where deptno = 10 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill +Sebastian +Theodore +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name from emps group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill +Eric +Sebastian +Theodore +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out new file mode 100644 index 0000000000..6e8b2e3e65 --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out @@ -0,0 +1,796 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where depts.deptno > 10 +group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where depts.deptno > 10 +group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, empid from depts +join emps using (deptno) where depts.deptno > 10 +group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, empid from depts +join emps using (deptno) where depts.deptno > 10 +group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where emps.deptno > 10 +group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno from emps +join depts using (deptno) where emps.deptno > 10 +group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps +join depts using (deptno) where depts.deptno > 20 +group by empid, depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 15) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno, emps.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 +10 +10 +10 +20 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, emps.empid from depts +join emps using (deptno) where emps.empid > 10 +group by depts.deptno, emps.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 15) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.deptno from depts +join emps using (deptno) where emps.empid > 15 +group by depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 +20 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.name, dependents.name as name2, emps.deptno, depts.deptno as deptno2, dependents.empid +from depts, dependents, emps +where depts.deptno > 10 +group by depts.name, dependents.name, emps.deptno, depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.name, dependents.name as name2, emps.deptno, depts.deptno as deptno2, dependents.empid +from depts, dependents, emps +where depts.deptno > 10 +group by depts.name, dependents.name, emps.deptno, depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 8 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno = deptno2) and (name = name2)) (type: boolean) + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out new file mode 100644 index 0000000000..e3bd233177 --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out @@ -0,0 +1,576 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from emps +join depts using (deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid > 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid > 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToInteger(_c0) > 1) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_c0) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid deptno from emps +join depts using (deptno) where empid > 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid deptno from emps +join depts using (deptno) where empid > 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +100 +110 +150 +200 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select cast(empid as BIGINT) from emps +join depts using (deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToInteger(_c0) = 1) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid deptno from emps +join depts using (deptno) where empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out new file mode 100644 index 0000000000..7301571cbf --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out @@ -0,0 +1,944 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name, count(*) as c, sum(empid) as s +from emps group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name, count(*) as c, sum(empid) as s +from emps group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), c (type: bigint), s (type: bigint) + outputColumnNames: name, c, s + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(c), sum(s) + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name, count(*) as c, sum(empid) as s +from emps group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name, count(*) as c, sum(empid) as s +from emps group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Bill 2 210 +Eric 1 200 +Sebastian 1 150 +Theodore 1 110 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, salary, count(*) as c, sum(empid) as s +from emps group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary +PREHOOK: type: QUERY +POSTHOOK: query: explain +select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: salary (type: float), name (type: varchar(256)), s (type: bigint), c (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select salary, name, sum(empid) as s, count(*) as c +from emps group by name, salary +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10000.0 Bill 210 2 +10000.0 Theodore 110 1 +7000.0 Sebastian 150 1 +8000.0 Eric 200 1 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, emps.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, emps.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), c (type: bigint), s (type: bigint) + outputColumnNames: deptno, c, s + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(c), sum(s) + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 4 470 +20 1 200 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno >= 10 group by empid, emps.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, emps.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno >= 10 group by empid, emps.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.deptno, sum(empid) as s +from emps join depts using (deptno) +where emps.deptno > 10 group by depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +20 200 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) + 1 as c, sum(empid) as s +from emps join depts using (deptno) +where depts.deptno >= 10 group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) + 1 as c, sum(empid) as s +from emps join depts using (deptno) +where depts.deptno >= 10 group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), s (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.deptno, sum(empid) + 1 as s +from emps join depts using (deptno) +where depts.deptno > 10 group by depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +20 201 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +group by depts.name +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +group by depts.name +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), s (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col1) + keys: _col2 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), s (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.deptno, count(distinct salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out new file mode 100644 index 0000000000..bab3cacc2c --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out @@ -0,0 +1,1522 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +POSTHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +PREHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +POSTHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary from emps where deptno > 15 group by name, deptno, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary from emps where deptno > 15 group by name, deptno, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps where deptno >= 20 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps where deptno >= 20 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno >= 20) (type: boolean) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps where deptno >= 20 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name from emps where deptno >= 20 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Eric +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select name, sum(empid) as s +from emps where deptno > 15 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name, sum(empid) as s +from emps where deptno > 15 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 15) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), s (type: bigint) + outputColumnNames: name, s + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name, sum(empid) as s +from emps where deptno > 15 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select name, sum(empid) as s +from emps where deptno > 15 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +Eric 200 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 19) and (deptno > 11)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select deptno from emps group by deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select deptno from emps group by deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select deptno from emps group by deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select deptno from emps group by deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 +20 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid, depts.deptno, count(*) as c, sum(empid) as s +from emps join depts using (deptno) +group by empid, depts.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), empid (type: int), s (type: bigint), c (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select deptno, empid, sum(empid) as s, count(*) as c +from emps group by empid, deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +10 100 100 1 +10 110 220 2 +10 150 150 1 +20 200 200 1 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), s (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select dependents.empid, emps.deptno, sum(salary) as s +from emps +join dependents on (emps.empid = dependents.empid) +group by dependents.empid, emps.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), s (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col1 (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select depts.name, sum(salary) as s +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +group by depts.name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: create materialized view mv1 enable rewrite as +select a.empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select a.empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: create materialized view mv1 enable rewrite as +select a.empid, a.deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select a.empid, a.deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select a.empid from +(select * from emps where empid = 1) a +join dependents on (a.empid = dependents.empid) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select empid deptno from +(select * from emps where empid = 1) a +join depts on (a.deptno = depts.deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps where empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps where empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select empid from emps where empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps where empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +Warning: Shuffle Join JOIN[16][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.empid = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno from emps +join depts a on (emps.deptno=a.deptno) +join depts b on (emps.deptno=b.deptno) +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.empid from emps +join dependents on (emps.empid = dependents.empid) +where emps.name = 'Sebastian' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out new file mode 100644 index 0000000000..017d793283 --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out @@ -0,0 +1,830 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +POSTHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +PREHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +POSTHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 10 group by name, deptno, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 10 group by name, deptno, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary +PREHOOK: type: QUERY +POSTHOOK: query: explain +select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: salary (type: float), s (type: bigint) + outputColumnNames: salary, s + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: salary (type: float) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: float) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: float), (_col1 + 1L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select salary, sum(empid) + 1 as s +from emps where deptno > 10 group by salary +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +8000.0 201 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select name, deptno, salary, count(*) + 1 as c, sum(empid) as s +from emps where deptno >= 15 group by name, deptno, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary +PREHOOK: type: QUERY +POSTHOOK: query: explain +select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 15) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: salary (type: float), s (type: bigint) + outputColumnNames: salary, s + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: salary (type: float) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: float) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (_col0 + 1.0) (type: float), (_col1 + 1L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select salary + 1, sum(empid) + 1 as s +from emps where deptno > 15 group by salary +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +8001.0 201 +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from emps +join depts on (emps.deptno = depts.deptno) +join dependents on (depts.name = dependents.name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.name +from emps +join depts on (emps.deptno = depts.deptno) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: locations + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col1 (type: varchar(256)) + outputColumnNames: _col2 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno, emps.name as name1, emps.salary, emps.commission, dependents.name as name2 +from emps join dependents on (emps.empid = dependents.empid) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select emps.empid, emps.deptno, emps.name as name1, emps.salary, emps.commission, dependents.name as name2 +from emps join dependents on (emps.empid = dependents.empid) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill' +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToString(name1) = 'Bill') (type: boolean) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col1 (type: int), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill' +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select emps.empid, dependents.empid, emps.deptno +from emps +join dependents on (emps.empid = dependents.empid) +join depts a on (emps.deptno=a.deptno) +where emps.name = 'Bill' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out new file mode 100644 index 0000000000..486a50d55a --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out @@ -0,0 +1,1036 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +POSTHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +PREHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +POSTHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 +group by depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 +group by depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-1 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno <= 11) and (deptno > 10) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno <= 11) and (deptno > 10)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + TableScan + alias: locations + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col1 (type: varchar(256)) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col3 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid, count(emps.salary) as s +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid, count(emps.salary) as s +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-1 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2, Stage-7 + Stage-7 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), salary (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: float) + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + TableScan + alias: locations + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Reduce Output Operator + key expressions: _col3 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col3 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: float) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col3 (type: varchar(256)) + outputColumnNames: _col0, _col4 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col4) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + TableScan + Union + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), s (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno >= 10 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno >= 10 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 10) and (deptno > 0) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 10) and (deptno > 0)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col3 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out new file mode 100644 index 0000000000..1ca06d3cc4 --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out @@ -0,0 +1,642 @@ +PREHOOK: query: create table if not exists source_table_001 ( +MY_DATE date, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@source_table_001 +POSTHOOK: query: create table if not exists source_table_001 ( +MY_DATE date, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source_table_001 +PREHOOK: query: insert into table source_table_001 + values ('2010-10-10', 1, 1, 'env', 1, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@source_table_001 +POSTHOOK: query: insert into table source_table_001 + values ('2010-10-10', 1, 1, 'env', 1, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@source_table_001 +POSTHOOK: Lineage: source_table_001.down_volume SCRIPT [] +POSTHOOK: Lineage: source_table_001.environment SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_date SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_id SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_id2 SCRIPT [] +POSTHOOK: Lineage: source_table_001.up_volume SCRIPT [] +PREHOOK: query: analyze table source_table_001 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@source_table_001 +PREHOOK: Output: default@source_table_001 +#### A masked pattern was here #### +POSTHOOK: query: analyze table source_table_001 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@source_table_001 +POSTHOOK: Output: default@source_table_001 +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001 +PREHOOK: Output: database:default +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source_table_001_mv +PREHOOK: query: analyze table source_table_001_mv compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +POSTHOOK: query: analyze table source_table_001_mv compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +LIMIT 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +LIMIT 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain +select +1, +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +1, +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int), down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 163 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) + 0 AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) + 0 AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (down_volume_sum + 0L) (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +ORDER BY A.MY_ID2 +LIMIT 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +ORDER BY A.MY_ID2 +LIMIT 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: bigint) + sort order: + + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: bigint), _col1 (type: date), _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: date), KEY.reducesinkkey0 (type: bigint), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +distinct A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +distinct A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: my_date, my_id2, environment + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: my_date (type: date), my_id2 (type: bigint), environment (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: date), KEY._col1 (type: bigint), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) + SUM(A.UP_VOLUME) AS TOTAL_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) + SUM(A.UP_VOLUME) AS TOTAL_VOLUME_BYTES, +A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 167 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 167 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (down_volume_sum + up_volume_sum) (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: down_volume_sum + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(down_volume_sum) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +TO_DATE('2010-01-10') +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +TO_DATE('2010-01-10') +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: down_volume_sum + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(down_volume_sum) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), DATE'2010-01-10' (type: date) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_DATE +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +A.MY_DATE +FROM source_table_001 AS A +where A.MY_DATE=TO_DATE('2010-01-10') +group by A.MY_DATE +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: DATE'2010-01-10' (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: bigint), DATE'2010-01-10' (type: date) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop materialized view source_table_001_mv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: drop materialized view source_table_001_mv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out new file mode 100644 index 0000000000..3120e0db7a --- /dev/null +++ b/ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out @@ -0,0 +1,361 @@ +PREHOOK: query: create table if not exists source_table_001 ( +MY_DATE timestamp, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@source_table_001 +POSTHOOK: query: create table if not exists source_table_001 ( +MY_DATE timestamp, +MY_ID bigint, +MY_ID2 bigint, +ENVIRONMENT string, +DOWN_VOLUME bigint, +UP_VOLUME bigint +) +stored AS ORC +TBLPROPERTIES("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source_table_001 +PREHOOK: query: insert into table source_table_001 + values ('2010-10-10 00:00:00', 1, 1, 'env', 1, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@source_table_001 +POSTHOOK: query: insert into table source_table_001 + values ('2010-10-10 00:00:00', 1, 1, 'env', 1, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@source_table_001 +POSTHOOK: Lineage: source_table_001.down_volume SCRIPT [] +POSTHOOK: Lineage: source_table_001.environment SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_date SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_id SCRIPT [] +POSTHOOK: Lineage: source_table_001.my_id2 SCRIPT [] +POSTHOOK: Lineage: source_table_001.up_volume SCRIPT [] +PREHOOK: query: analyze table source_table_001 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@source_table_001 +PREHOOK: Output: default@source_table_001 +#### A masked pattern was here #### +POSTHOOK: query: analyze table source_table_001 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@source_table_001 +POSTHOOK: Output: default@source_table_001 +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001 +PREHOOK: Output: database:default +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,A.MY_DATE,A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source_table_001_mv +PREHOOK: query: analyze table source_table_001_mv compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +POSTHOOK: query: analyze table source_table_001_mv compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_hour(my_date) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP MATERIALIZED VIEW source_table_001_mv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: DROP MATERIALIZED VIEW source_table_001_mv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv +PREHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001 +PREHOOK: Output: database:default +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: CREATE MATERIALIZED VIEW source_table_001_mv ENABLE REWRITE AS +SELECT +SUM(A.DOWN_VOLUME) AS DOWN_VOLUME_SUM, +SUM(A.UP_VOLUME) AS UP_VOLUME_SUM, +A.MY_ID,FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +from source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source_table_001_mv +PREHOOK: query: analyze table source_table_001_mv compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +POSTHOOK: query: analyze table source_table_001_mv compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv +#### A masked pattern was here #### +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to day),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to day) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to day),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to day) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_day(_c3) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to hour),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to hour) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), _c3 (type: timestamp), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to second),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to second) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select +SUM(A.DOWN_VOLUME) AS DOWNLOAD_VOLUME_BYTES, +FLOOR(A.MY_DATE to second),A.MY_ID2,A.ENVIRONMENT +FROM source_table_001 AS A +group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,FLOOR(A.MY_DATE to second) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_second(my_date) (type: timestamp), down_volume (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col4) + keys: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP MATERIALIZED VIEW source_table_001_mv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@source_table_001_mv +PREHOOK: Output: default@source_table_001_mv +POSTHOOK: query: DROP MATERIALIZED VIEW source_table_001_mv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@source_table_001_mv +POSTHOOK: Output: default@source_table_001_mv diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3e1fea9d4f..de941729e1 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1276,6 +1276,8 @@ public boolean dropTable(String dbName, String tableName) throws MetaException, private boolean dropCreationMetadata(String dbName, String tableName) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean success = false; + dbName = normalizeIdentifier(dbName); + tableName = normalizeIdentifier(tableName); try { openTransaction(); MCreationMetadata mcm = getCreationMetadata(dbName, tableName);