diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 807f159daa..7cad6076e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -58,6 +58,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.calcite.rel.RelNode; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -169,8 +170,10 @@ import org.apache.hadoop.hive.ql.metadata.formatting.TextMetaDataTable; import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.CalcitePlanner; import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; +import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.PreInsertTableDesc; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -1257,6 +1260,22 @@ private int alterMaterializedView(Hive db, AlterMaterializedViewDesc alterMVDesc // This is a noop, return successfully return 0; } + if (alterMVDesc.isRewriteEnable()) { + try { + final CalcitePlanner planner = ParseUtils.getAnalyzer(conf); + final RelNode plan = planner.genLogicalPlan(ParseUtils.parse(mv.getViewExpandedText())); + if (plan == null) { + throw new HiveException("Cannot enable automatic rewriting for materialized view. " + + "Materialized view definition could not be parsed by CBO."); + } + if (!planner.isValidAutomaticRewritingMaterialization()) { + throw new HiveException("Cannot enable rewriting for materialized view. " + + planner.getInvalidAutomaticRewritingMaterializationReason()); + } + } catch (Exception e) { + throw new HiveException(e); + } + } mv.setRewriteEnabled(alterMVDesc.isRewriteEnable()); break; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptAutomaticRewritingMaterializationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptAutomaticRewritingMaterializationValidator.java new file mode 100644 index 0000000000..ee3e24565c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptAutomaticRewritingMaterializationValidator.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.TableFunctionScan; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalExchange; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalIntersect; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalMatch; +import org.apache.calcite.rel.logical.LogicalMinus; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.Util; + +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; + +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Checks whether the plan is supported by the automatic rewriting algorithm. + * For instance, a materialized view definition should be SPJA query and it + * cannot contain outer joins. + */ +public class HiveRelOptAutomaticRewritingMaterializationValidator extends HiveRelShuttleImpl { + static final Logger LOG = LoggerFactory.getLogger(HiveRelOptAutomaticRewritingMaterializationValidator.class); + + protected String invalidReason; + + public void validate(RelNode relNode) { + try { + relNode.accept(this); + } catch (Util.FoundOne e) { + // Can ignore - the check failed. + } + } + + @Override + public RelNode visit(TableScan scan) { + return super.visit(scan); + } + + @Override + public RelNode visit(HiveProject project) { + return super.visit(project); + } + + @Override + public RelNode visit(HiveFilter filter) { + return super.visit(filter); + } + + @Override + public RelNode visit(HiveJoin join) { + if (join.getJoinType() != JoinRelType.INNER) { + fail(join.getJoinType() + " join type is not supported by rewriting algorithm."); + } + return super.visit(join); + } + + @Override + public RelNode visit(HiveAggregate aggregate) { + return super.visit(aggregate); + } + + @Override + public RelNode visit(RelNode node) { + // Fall-back for an unexpected RelNode type + return fail(node); + } + + @Override + public RelNode visit(TableFunctionScan scan) { + // Not supported - fail + return fail(scan); + } + + @Override + public RelNode visit(LogicalValues values) { + // Not expected to be encountered for Hive - fail + return fail(values); + } + + @Override + public RelNode visit(LogicalFilter filter) { + // Not expected to be encountered for Hive - fail + return fail(filter); + } + + @Override + public RelNode visit(LogicalProject project) { + // Not expected to be encountered for Hive - fail + return fail(project); + } + + @Override + public RelNode visit(LogicalJoin join) { + // Not expected to be encountered for Hive - fail + return fail(join); + } + + @Override + public RelNode visit(LogicalCorrelate correlate) { + // Not expected to be encountered for Hive - fail + return fail(correlate); + } + + @Override + public RelNode visit(LogicalUnion union) { + // Not expected to be encountered for Hive - fail + return fail(union); + } + + @Override + public RelNode visit(LogicalIntersect intersect) { + // Not expected to be encountered for Hive - fail + return fail(intersect); + } + + @Override + public RelNode visit(LogicalMinus minus) { + // Not expected to be encountered for Hive - fail + return fail(minus); + } + + @Override + public RelNode visit(LogicalAggregate aggregate) { + // Not expected to be encountered for Hive - fail + return fail(aggregate); + } + + @Override + public RelNode visit(LogicalMatch match) { + // Not expected to be encountered for Hive - fail + return fail(match); + } + + @Override + public RelNode visit(LogicalSort sort) { + // Not expected to be encountered for Hive - fail + return fail(sort); + } + + @Override + public RelNode visit(LogicalExchange exchange) { + // Not expected to be encountered for Hive - fail + return fail(exchange); + } + + private void fail(String reason) { + setInvalidReason(reason); + throw Util.FoundOne.NULL; + } + + private RelNode fail(RelNode node) { + setInvalidReason("Unsupported RelNode type " + node.getRelTypeName() + + " encountered in the query plan"); + throw Util.FoundOne.NULL; + } + + public String getInvalidReason() { + return invalidReason; + } + + public void setInvalidReason(String invalidReason) { + this.invalidReason = invalidReason; + } + + public boolean isValid() { + return invalidReason == null; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptCachedQueryValidator.java similarity index 90% rename from ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java rename to ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptCachedQueryValidator.java index df216e7555..784351b168 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptCachedQueryValidator.java @@ -18,10 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; -import java.util.List; - import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -39,11 +36,8 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; - import org.apache.hadoop.hive.metastore.TableType; - import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -54,7 +48,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,12 +57,12 @@ * - References to temporary or external tables * - References to non-determinisitc functions. */ -public class HiveRelOpMaterializationValidator extends HiveRelShuttleImpl { - static final Logger LOG = LoggerFactory.getLogger(HiveRelOpMaterializationValidator.class); +public class HiveRelOptCachedQueryValidator extends HiveRelShuttleImpl { + static final Logger LOG = LoggerFactory.getLogger(HiveRelOptCachedQueryValidator.class); - protected String invalidMaterializationReason; + protected String invalidReason; - public void validateQueryMaterialization(RelNode relNode) { + public void validate(RelNode relNode) { try { relNode.accept(this); } catch (Util.FoundOne e) { @@ -254,12 +247,12 @@ private RelNode visit(HiveIntersect intersect) { } private void fail(String reason) { - setInvalidMaterializationReason(reason); + setInvalidReason(reason); throw Util.FoundOne.NULL; } private RelNode fail(RelNode node) { - setInvalidMaterializationReason("Unsupported RelNode type " + node.getRelTypeName() + + setInvalidReason("Unsupported RelNode type " + node.getRelTypeName() + " encountered in the query plan"); throw Util.FoundOne.NULL; } @@ -271,15 +264,15 @@ private void checkExpr(RexNode expr) { } } - public String getInvalidMaterializationReason() { - return invalidMaterializationReason; + public String getInvalidReason() { + return invalidReason; } - public void setInvalidMaterializationReason(String invalidMaterializationReason) { - this.invalidMaterializationReason = invalidMaterializationReason; + public void setInvalidReason(String invalidReason) { + this.invalidReason = invalidReason; } - public boolean isValidMaterialization() { - return invalidMaterializationReason == null; + public boolean isValid() { + return invalidReason == null; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 22f3266c87..0e870a5c74 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -148,7 +148,8 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOpMaterializationValidator; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptAutomaticRewritingMaterializationValidator; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptCachedQueryValidator; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -1731,16 +1732,6 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu } perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Plan generation"); - // Validate query materialization (materialized views, query results caching. - // This check needs to occur before constant folding, which may remove some - // function calls from the query plan. - HiveRelOpMaterializationValidator matValidator = new HiveRelOpMaterializationValidator(); - matValidator.validateQueryMaterialization(calciteGenPlan); - if (!matValidator.isValidMaterialization()) { - String reason = matValidator.getInvalidMaterializationReason(); - setInvalidQueryMaterializationReason(reason); - } - // Create executor RexExecutor executorProvider = new HiveRexExecutorImpl(optCluster); calciteGenPlan.getCluster().getPlanner().setExecutor(executorProvider); @@ -1766,6 +1757,31 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu calciteGenPlan = HiveRelDecorrelator.decorrelateQuery(calciteGenPlan); LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calciteGenPlan)); + // Validate query materialization for query results caching. This check needs + // to occur before constant folding, which may remove some function calls + // from the query plan. + // In addition, if it is a materialized view creation and we are enabling it + // for rewriting, it should pass all checks done for query results caching + // and on top of that we should check that it only contains operators that + // are supported by the rewriting algorithm. + HiveRelOptCachedQueryValidator cachedQueryValidator = new HiveRelOptCachedQueryValidator(); + cachedQueryValidator.validate(calciteGenPlan); + if (!cachedQueryValidator.isValid()) { + String reason = cachedQueryValidator.getInvalidReason(); + setInvalidQueryMaterializationReason(reason); + // Automatic query rewriting should meet all cached query validator conditions, + // hence if it does not, we can add the reason here and skip scanning it again + setInvalidAutomaticRewritingMaterializationReason(reason); + } else { + // It passed first test, check whether is is valid for automatic rewriting + HiveRelOptAutomaticRewritingMaterializationValidator mvValidator = + new HiveRelOptAutomaticRewritingMaterializationValidator(); + mvValidator.validate(calciteGenPlan); + if (!mvValidator.isValid()) { + setInvalidAutomaticRewritingMaterializationReason(mvValidator.getInvalidReason()); + } + } + // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, mdProvider.getMetadataProvider(), executorProvider); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index be1c59f932..5d229b65b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -555,7 +555,7 @@ public static RelNode parseQuery(HiveConf conf, String viewQuery) return analyzer.getResultSchema(); } - private static CalcitePlanner getAnalyzer(HiveConf conf) throws SemanticException, IOException { + public static CalcitePlanner getAnalyzer(HiveConf conf) throws SemanticException, IOException { final QueryState qs = new QueryState.Builder().withHiveConf(conf).build(); CalcitePlanner analyzer = new CalcitePlanner(qs); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6a6e6c3639..2b9ba4f41c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -398,6 +398,7 @@ HiveParser.TOK_DISTRIBUTEBY, HiveParser.TOK_SORTBY); private String invalidQueryMaterializationReason; + private String invalidAutomaticRewritingMaterializationReason; private static final CommonToken SELECTDI_TOKEN = new ImmutableCommonToken(HiveParser.TOK_SELECTDI, "TOK_SELECTDI"); @@ -13714,6 +13715,17 @@ private void validateCreateView() } } + if (createVwDesc.isMaterialized() && createVwDesc.isRewriteEnabled()) { + if (!ctx.isCboSucceeded()) { + throw new SemanticException("Cannot enable automatic rewriting for materialized view. " + + "Materialized view definition could not be parsed by CBO."); + } + if (!isValidAutomaticRewritingMaterialization()) { + throw new SemanticException("Cannot enable rewriting for materialized view. " + + getInvalidAutomaticRewritingMaterializationReason()); + } + } + // ALTER VIEW AS SELECT requires the view must exist if (createVwDesc.getIsAlterViewAs() && oldView == null) { String viewNotExistErrorMsg = @@ -15129,6 +15141,20 @@ public ColsAndTypes(String cols, String colTypes) { public String colTypes; } + public String getInvalidAutomaticRewritingMaterializationReason() { + return invalidAutomaticRewritingMaterializationReason; + } + + public void setInvalidAutomaticRewritingMaterializationReason( + String invalidAutomaticRewritingMaterializationReason) { + this.invalidAutomaticRewritingMaterializationReason = + invalidAutomaticRewritingMaterializationReason; + } + + public boolean isValidAutomaticRewritingMaterialization() { + return (invalidAutomaticRewritingMaterializationReason == null); + } + public String getInvalidQueryMaterializationReason() { return invalidQueryMaterializationReason; } diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q new file mode 100644 index 0000000000..d28ff0b6fd --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q @@ -0,0 +1,11 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view as select a, b, c from cmv_basetable sort by a; diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q new file mode 100644 index 0000000000..ea360ebd06 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q @@ -0,0 +1,13 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a; + +alter materialized view cmv_mat_view enable rewrite; diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q new file mode 100644 index 0000000000..204c264812 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q @@ -0,0 +1,11 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b); diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q new file mode 100644 index 0000000000..7050f1ac9a --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q @@ -0,0 +1,13 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b); + +alter materialized view cmv_mat_view enable rewrite; diff --git a/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out new file mode 100644 index 0000000000..255056d0c2 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +FAILED: SemanticException Cannot enable automatic rewriting for materialized view. Materialized view definition could not be parsed by CBO. diff --git a/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out new file mode 100644 index 0000000000..84660a6d31 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +PREHOOK: query: create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: alter materialized view cmv_mat_view enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Cannot enable automatic rewriting for materialized view. Materialized view definition could not be parsed by CBO. diff --git a/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out new file mode 100644 index 0000000000..f3341bf585 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +FAILED: SemanticException Cannot enable rewriting for materialized view. LEFT join type is not supported by rewriting algorithm. diff --git a/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out new file mode 100644 index 0000000000..0b72e3cb83 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out @@ -0,0 +1,37 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: alter materialized view cmv_mat_view enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Cannot enable rewriting for materialized view. LEFT join type is not supported by rewriting algorithm.